pax_global_header00006660000000000000000000000064152557631050014523gustar00rootroot0000000000000052 comment=1330414143be82209e0a0860d2b9f8ab567c94b5 kanzi-cpp-2.6.0/000077500000000000000000000000001525576310500134245ustar00rootroot00000000000000kanzi-cpp-2.6.0/.github/000077500000000000000000000000001525576310500147645ustar00rootroot00000000000000kanzi-cpp-2.6.0/.github/scripts/000077500000000000000000000000001525576310500164535ustar00rootroot00000000000000kanzi-cpp-2.6.0/.github/scripts/msvc_build.cmd000066400000000000000000000036311525576310500212720ustar00rootroot00000000000000@echo off setlocal EnableExtensions if "%~4"=="" ( echo Usage: msvc_build.cmd ROOT OUT_DIR TARGET ARCH echo TARGET is testTransforms or kanzi exit /b 2 ) set "ROOT=%~f1" set "OUT_DIR=%~f2" set "TARGET=%~3" set "ARCH=%~4" set "OBJ_DIR=%OUT_DIR%\obj-%TARGET%-%ARCH%" for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do set "VSINSTALL=%%i" if "%VSINSTALL%"=="" ( echo Visual Studio installation not found exit /b 1 ) call "%VSINSTALL%\Common7\Tools\VsDevCmd.bat" -arch=%ARCH% -host_arch=x64 if errorlevel 1 exit /b %errorlevel% if not exist "%OUT_DIR%" mkdir "%OUT_DIR%" if not exist "%OBJ_DIR%" mkdir "%OBJ_DIR%" pushd "%ROOT%" if errorlevel 1 exit /b %errorlevel% git ls-files "src/*.cpp" "src/*/*.cpp" | findstr /v /i /c:"src/test/" /c:"src/app/" > "%OUT_DIR%\kanzi_lib_sources.rsp" if errorlevel 1 exit /b %errorlevel% git ls-files "src/*.cpp" "src/*/*.cpp" | findstr /v /i /c:"src/test/" > "%OUT_DIR%\kanzi_cli_sources.rsp" if errorlevel 1 exit /b %errorlevel% if /i "%TARGET%"=="testTransforms" goto build_testTransforms if /i "%TARGET%"=="kanzi" goto build_kanzi popd echo Unknown target: %TARGET% exit /b 2 :build_testTransforms ( echo src\test\TestTransforms.cpp type "%OUT_DIR%\kanzi_lib_sources.rsp" ) > "%OUT_DIR%\test_transform_sources.rsp" cl /nologo /EHsc /MT /O2 /DNDEBUG /D_CRT_SECURE_NO_WARNINGS /DTestTransforms_main=main /GR- /std:c++17 /Zc:__cplusplus /I "%ROOT%\src" /Fo"%OBJ_DIR%\\" /Fe"%OUT_DIR%\testTransforms.exe" @"%OUT_DIR%\test_transform_sources.rsp" set "RC=%ERRORLEVEL%" popd exit /b %RC% :build_kanzi cl /nologo /EHsc /MT /O2 /DNDEBUG /D_CRT_SECURE_NO_WARNINGS /GR- /std:c++17 /Zc:__cplusplus /I "%ROOT%\src" /Fo"%OBJ_DIR%\\" /Fe"%OUT_DIR%\kanzi.exe" @"%OUT_DIR%\kanzi_cli_sources.rsp" set "RC=%ERRORLEVEL%" popd exit /b %RC% kanzi-cpp-2.6.0/.github/scripts/stream_roundtrip.py000066400000000000000000000070731525576310500224350ustar00rootroot00000000000000#!/usr/bin/env python3 import argparse import hashlib import random import subprocess import tempfile from pathlib import Path SAMPLE_SIZE = 4 * 1024 * 1024 CHUNK_SIZE = 256 * 1024 LEVELS = tuple(range(1, 10)) def sha256(path): digest = hashlib.sha256() with path.open("rb") as source: for chunk in iter(lambda: source.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def write_random(path): rng = random.Random(0x4B414E5A49) remaining = SAMPLE_SIZE with path.open("wb") as target: while remaining > 0: size = min(CHUNK_SIZE, remaining) target.write(bytes(rng.getrandbits(8) for _ in range(size))) remaining -= size def write_structured(path): line = ( b"kanzi stream transform check 0123456789 " b"abcdefghijklmnopqrstuvwxyz repeated fields ROLZ RANK SRT " + b"A" * 96 + b"\n" ) chunk = (line * ((CHUNK_SIZE // len(line)) + 1))[:CHUNK_SIZE] remaining = SAMPLE_SIZE with path.open("wb") as target: while remaining > 0: target.write(chunk[: min(CHUNK_SIZE, remaining)]) remaining -= CHUNK_SIZE def write_mixed(path): rng = random.Random(0x53545245414D) chunks = SAMPLE_SIZE // CHUNK_SIZE with path.open("wb") as target: for block in range(chunks): mode = block % 4 if mode == 0: data = bytes((idx + block) & 255 for idx in range(CHUNK_SIZE)) elif mode == 1: data = bytes(65 + (((idx // 257) + block) % 26) for idx in range(CHUNK_SIZE)) elif mode == 2: data = bytes(rng.getrandbits(8) for _ in range(CHUNK_SIZE)) else: data = b"\0" * CHUNK_SIZE target.write(data) def run(command): print("+ " + " ".join(str(part) for part in command), flush=True) subprocess.run(command, check=True) def main(): parser = argparse.ArgumentParser() parser.add_argument("kanzi", type=Path) args = parser.parse_args() kanzi = args.kanzi.resolve() if not kanzi.is_file(): raise SystemExit(f"Missing Kanzi executable: {kanzi}") with tempfile.TemporaryDirectory(prefix="kanzi-stream-") as tmp_dir: root = Path(tmp_dir) samples = { "random": root / "random-4m.bin", "structured": root / "structured-4m.txt", "mixed": root / "mixed-4m.bin", } writers = { "random": write_random, "structured": write_structured, "mixed": write_mixed, } for name, writer in writers.items(): writer(samples[name]) for name, source in samples.items(): source_hash = sha256(source) for level in LEVELS: compressed = root / f"{name}-l{level}.knz" decoded = root / f"{name}-l{level}.out" run([str(kanzi), "-c", "-i", str(source), "-o", str(compressed), "-f", "-b", "1m", "-l", str(level), "-x32", "-j", "1", "-v", "1"]) run([str(kanzi), "-d", "-i", str(compressed), "-o", str(decoded), "-f", "-j", "1", "-v", "1"]) decoded_hash = sha256(decoded) if decoded_hash != source_hash: raise SystemExit(f"SHA-256 mismatch for {name} level {level}") print( f"{name} level {level}: {source.stat().st_size} => " f"{compressed.stat().st_size}, SHA-256 match", flush=True, ) if __name__ == "__main__": main() kanzi-cpp-2.6.0/.github/scripts/transform_regression.py000066400000000000000000000051321525576310500233010ustar00rootroot00000000000000#!/usr/bin/env python3 import argparse import subprocess import tempfile from pathlib import Path COMMANDS = ( ("ROLZ", ("-type=ROLZ", "-noperf")), ("SRT", ("-type=SRT", "-noperf")), ("RANK", ("-type=RANK", "-noperf")), ("all", ("-type=all", "-noperf")), ) def print_tail(path, lines=80): try: content = path.read_text(errors="replace").splitlines() except OSError as exc: print(f"Could not read log {path}: {exc}") return tail = content[-lines:] if tail: print(f"--- tail of {path.name} ---") print("\n".join(tail)) print("--- end tail ---") def run_case(executable, name, args, timeout): log_file = Path(tempfile.gettempdir()) / f"kanzi-transform-{name}.log" command = [str(executable), *args] print("+ " + " ".join(command), flush=True) with log_file.open("w", encoding="utf-8", errors="replace") as log: try: result = subprocess.run( command, stdout=log, stderr=subprocess.STDOUT, timeout=timeout, check=False, ) return result.returncode, log_file except subprocess.TimeoutExpired: log.write(f"\nTIMEOUT after {timeout} seconds\n") return 124, log_file def main(): parser = argparse.ArgumentParser() parser.add_argument("executable", type=Path) parser.add_argument("--expect", choices=("pass", "fail"), required=True) parser.add_argument("--timeout", type=int, default=300) args = parser.parse_args() executable = args.executable.resolve() if not executable.is_file(): raise SystemExit(f"Missing test executable: {executable}") failures = [] for name, command_args in COMMANDS: code, log_file = run_case(executable, name, command_args, args.timeout) print(f"{name}: exit {code}, log {log_file}", flush=True) if code != 0: failures.append((name, code, log_file)) print_tail(log_file) if args.expect == "pass": if failures: failed = ", ".join(f"{name}={code}" for name, code, _ in failures) raise SystemExit(f"Expected all transform cases to pass, got {failed}") print("All transform regression cases passed", flush=True) return if not failures: raise SystemExit("Expected the baseline to reproduce at least one transform failure") failed = ", ".join(f"{name}={code}" for name, code, _ in failures) print(f"Baseline reproduced transform failure(s): {failed}", flush=True) if __name__ == "__main__": main() kanzi-cpp-2.6.0/.github/workflows/000077500000000000000000000000001525576310500170215ustar00rootroot00000000000000kanzi-cpp-2.6.0/.github/workflows/c-cpp.yml000066400000000000000000000005421525576310500205470ustar00rootroot00000000000000name: C/C++ CI on: push: branches: [ master ] pull_request: branches: [ master ] jobs: build: strategy: matrix: os: [ ubuntu-latest, macos-latest ] compiler: [ clang, gcc ] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v6.0.1 - name: make run: cd src && make clean && make all kanzi-cpp-2.6.0/.github/workflows/codeql.yml000066400000000000000000000056211525576310500210170ustar00rootroot00000000000000# For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: [ "master" ] pull_request: # The branches below must be a subset of the branches above branches: [ "master" ] schedule: - cron: '18 0 * * 5' jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'cpp' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Use only 'java' to analyze code written in Java, Kotlin or both # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository uses: actions/checkout@v6.0.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v4.31.9 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v4.31.9 # ℹī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun # If the Autobuild fails above, remove it and uncomment the following three lines. # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. # - run: | # echo "Run, Build Application using script" # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4.31.9 with: category: "/language:${{matrix.language}}" kanzi-cpp-2.6.0/.github/workflows/kanzi-quality.yml000066400000000000000000000071321525576310500223510ustar00rootroot00000000000000name: Kanzi quality on: push: branches: [ master ] pull_request: branches: [ master ] workflow_dispatch: inputs: baseline_ref: description: Known-bad ref used by the MSVC contrast job required: false default: f583957239707a6913a2a9994527bb1d3f623aa7 env: KANZI_REGRESSION_BASE: f583957239707a6913a2a9994527bb1d3f623aa7 jobs: windows-regression-contrast: name: windows-2025-vs2026 / msvc x64 baseline contrast runs-on: windows-2025-vs2026 steps: - uses: actions/checkout@v6.0.1 with: fetch-depth: 0 - name: Select known-bad baseline shell: pwsh run: | $baseline = "${{ github.event.inputs.baseline_ref }}" if ([string]::IsNullOrWhiteSpace($baseline)) { $baseline = "${{ env.KANZI_REGRESSION_BASE }}" } "BASELINE_REF=$baseline" >> $env:GITHUB_ENV - name: Build baseline and current testTransforms shell: cmd run: | git worktree add "%RUNNER_TEMP%\baseline" "%BASELINE_REF%" call .github\scripts\msvc_build.cmd "%RUNNER_TEMP%\baseline" "%RUNNER_TEMP%\baseline-msvc-x64" testTransforms x64 if errorlevel 1 exit /b %errorlevel% call .github\scripts\msvc_build.cmd "%GITHUB_WORKSPACE%" "%RUNNER_TEMP%\head-msvc-x64" testTransforms x64 if errorlevel 1 exit /b %errorlevel% - name: Confirm baseline reproduces transform failure shell: cmd run: python .github\scripts\transform_regression.py --expect fail "%RUNNER_TEMP%\baseline-msvc-x64\testTransforms.exe" - name: Confirm current head fixes transform failure shell: cmd run: python .github\scripts\transform_regression.py --expect pass "%RUNNER_TEMP%\head-msvc-x64\testTransforms.exe" unix-transforms: name: ${{ matrix.os }} / ${{ matrix.cxx }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: include: - os: ubuntu-latest cc: gcc cxx: g++ - os: ubuntu-latest cc: clang cxx: clang++ - os: macos-latest cc: clang cxx: clang++ steps: - uses: actions/checkout@v6.0.1 - name: Build testTransforms and kanzi run: | cd src make clean make testTransforms kanzi CC=${{ matrix.cc }} CXX=${{ matrix.cxx }} - name: Run transform tests run: python3 .github/scripts/transform_regression.py --expect pass bin/testTransforms - name: Run stream round trips run: python3 .github/scripts/stream_roundtrip.py bin/kanzi windows-transforms: name: windows-2025-vs2026 / msvc ${{ matrix.arch }} runs-on: windows-2025-vs2026 strategy: fail-fast: false matrix: arch: [ x64, x86 ] steps: - uses: actions/checkout@v6.0.1 - name: Build and run transform tests shell: cmd run: | call .github\scripts\msvc_build.cmd "%GITHUB_WORKSPACE%" "%RUNNER_TEMP%\head-msvc-${{ matrix.arch }}" testTransforms ${{ matrix.arch }} if errorlevel 1 exit /b %errorlevel% python .github\scripts\transform_regression.py --expect pass "%RUNNER_TEMP%\head-msvc-${{ matrix.arch }}\testTransforms.exe" - name: Build and run stream round trips shell: cmd run: | call .github\scripts\msvc_build.cmd "%GITHUB_WORKSPACE%" "%RUNNER_TEMP%\head-msvc-${{ matrix.arch }}" kanzi ${{ matrix.arch }} if errorlevel 1 exit /b %errorlevel% python .github\scripts\stream_roundtrip.py "%RUNNER_TEMP%\head-msvc-${{ matrix.arch }}\kanzi.exe" kanzi-cpp-2.6.0/API_REFERENCE.md000066400000000000000000001032471525576310500157440ustar00rootroot00000000000000# Kanzi C++ API Reference This document describes the public integration APIs exposed by Kanzi C++: - The C ABI in `src/api/Compressor.hpp` and `src/api/Decompressor.hpp` - The C++ stream API in `src/io/CompressedOutputStream.hpp` and `src/io/CompressedInputStream.hpp` - Shared C++ extension interfaces for transforms, entropy codecs, bitstreams, contexts, events, and errors - The Python `ctypes` wrapper in `src/api/kanzi.py` Kanzi is a compressor, not an archive format. The APIs below operate on byte streams or blocks; callers remain responsible for file naming, metadata, multi-file packaging, and application-level lifecycle decisions. ## Contents - [Compatibility](#compatibility) - [Common Concepts](#common-concepts) - [C API](#c-api) - [C++ Stream API](#c-stream-api) - [Context Keys](#context-keys) - [Transforms](#transforms) - [Entropy Codecs](#entropy-codecs) - [Bitstream API](#bitstream-api) - [Events and Listeners](#events-and-listeners) - [Errors and Exceptions](#errors-and-exceptions) - [Python Wrapper](#python-wrapper) - [Integration Notes](#integration-notes) ## Compatibility The core library is written in portable C++ and supports multiple C++ standards. Some APIs expose extra functionality when the compiler supports it: - C++98 and later: core C++ stream and codec APIs - C++11 and later: move constructors/assignments in several value types - C++17 and later: `Context` stores values internally with `std::variant` - `CONCURRENCY_ENABLED`: multi-job compression/decompression and `ThreadPool` integration - MSVC 2008: some seek helpers are unavailable because the bitstream classes do not inherit `Seekable` on that compiler path The C API is exported with `extern "C"` and uses opaque contexts plus `FILE*`. ## Common Concepts ### Namespace and Types Most C++ APIs live in namespace `kanzi`. Common integer aliases are defined in `src/types.hpp`: | Alias | Meaning | | --- | --- | | `byte` | Byte type. `std::byte` when enabled with C++17 and `KNZ_BYTE_AS_STD_BYTE`, otherwise `uint8_t`. | | `int8`, `uint8`, `int16`, `uint16`, `int32`, `uint32` | Fixed-width integer aliases. | | `uint` | `uint32_t`. | | `int64`, `uint64` | 64-bit signed and unsigned aliases. | ### Transform and Entropy Names Names are case-insensitive when parsed through factories or the C API. Supported transform names: | Name | Description | | --- | --- | | `NONE` | No transform. | | `PACK` | Alias packing transform. | | `DNA` | DNA-specialized alias packing. | | `BWT` | Burrows-Wheeler transform block codec. | | `BWTS` | Burrows-Wheeler Scott transform. | | `LZ` | Default Lempel-Ziv transform. | | `LZX` | Lempel-Ziv extra variant. | | `LZP` | Lempel-Ziv predictive transform. | | `ROLZ` | Reduced-offset Lempel-Ziv transform. | | `ROLZX` | Extra ROLZ variant. | | `RLT` | Run-length transform. | | `ZRLT` | Zero run-length transform. | | `MTFT` | Move-to-front family transform. | | `RANK` | Rank transform. | | `SRT` | Sorted-rank transform. | | `TEXT` | Text/dictionary transform. | | `MM` | Multimedia/FSD transform. | | `EXE` | Executable-code transform. | | `UTF` | UTF transform. | Transforms can be chained with `+`, for example `BWT+RANK+ZRLT`. Up to 8 transform tokens are accepted. Supported entropy names: | Name | Description | | --- | --- | | `NONE` | No entropy coding. | | `HUFFMAN` | Huffman coding. | | `ANS0` | ANS range coding, order 0. | | `ANS1` | ANS range coding, order 1. | | `RANGE` | Range coding. | | `FPAQ` | Fast PAQ-style bit coding. | | `CM` | Context model. | | `TPAQ` | Tangelo PAQ. | | `TPAQX` | Tangelo PAQ extra. | ### Blocks Kanzi compresses data in blocks. General stream constraints: | Parameter | Constraint | | --- | --- | | Block size | `1024` to `1024 * 1024 * 1024` bytes | | Block size alignment | Multiple of 16 for compression streams | | Checksum size | `0`, `32`, or `64` | | Jobs | `1..64` when concurrency is enabled, exactly `1` otherwise | The C API rounds `blockSize` up to the next multiple of 16 during initialization. ### Headered vs Headerless Streams Default streams are headered. The compressed stream contains the transform, entropy codec, checksum size, block size, and bitstream version. Headerless mode omits this metadata. Callers must provide matching parameters to both the compressor and decompressor. Use headerless mode only when an outer protocol already carries the Kanzi stream parameters. ## C API Headers: ```c #include "api/Compressor.hpp" #include "api/Decompressor.hpp" ``` The C API returns `0` on success and a Kanzi error code on failure. Error codes are listed in [Errors and Exceptions](#errors-and-exceptions). ### C Compression Types ```c struct cContext; struct cData { char transform[64]; char entropy[16]; size_t blockSize; unsigned int jobs; int checksum; int headerless; }; ``` `cContext` is opaque. Do not allocate or inspect it directly. `cData` fields: | Field | Direction | Description | | --- | --- | --- | | `transform` | in/out | Transform name or chain. Validated and rewritten to canonical uppercase form by `initCompressor()`. | | `entropy` | in/out | Entropy codec name. Validated and rewritten to canonical uppercase form by `initCompressor()`. | | `blockSize` | in/out | Maximum input bytes accepted per `compress()` call. Rounded up to a multiple of 16 by `initCompressor()`. | | `jobs` | in | Maximum concurrent jobs. | | `checksum` | in | `0`, `32`, or `64`. | | `headerless` | in | Non-zero writes a headerless bitstream. | ### `getCompressorVersion` ```c unsigned int getCompressorVersion(void); ``` Returns the compressor C API version packed as: ```text (major << 16) | (minor << 8) | patch ``` Current header constants are `KANZI_COMP_VERSION_MAJOR`, `KANZI_COMP_VERSION_MINOR`, and `KANZI_COMP_VERSION_PATCH`. ### `initCompressor` ```c int initCompressor(struct cData* cParam, FILE* dst, struct cContext** ctx); ``` Initializes a compressor writing to `dst`. Rules: - `cParam`, `dst`, and `ctx` must be non-null. - `transform` and `entropy` must be null-terminated inside their fixed arrays. - `transform` and `entropy` are validated and normalized in place. - `blockSize` is rounded up to a multiple of 16. - On success, `*ctx` receives a new compressor context. - On failure, no compressor context should be used. Common return codes: | Code | Meaning | | --- | --- | | `0` | Success. | | `ERR_INVALID_PARAM` | Null argument, unterminated string, invalid parameter. | | `ERR_CREATE_COMPRESSOR` | Could not create the internal compressor stream. | ### `compress` ```c int compress(struct cContext* ctx, const unsigned char* src, size_t inSize, size_t* outSize); ``` Compresses one input block and writes compressed bytes to the destination `FILE*` provided at initialization. Rules: - `ctx` and `outSize` must be non-null. - `src` may be null only when `inSize == 0`. - `inSize` must not exceed the initialized `blockSize`. - `*outSize` is set to the number of bytes written during this call. - Zero-length input is accepted and reports zero output. The compressor owns no input memory. The caller may reuse or free `src` after the call returns. ### `disposeCompressor` ```c int disposeCompressor(struct cContext** ctx, size_t* outSize); ``` Closes the compressor, flushes remaining internal data, releases resources, and sets `*ctx` to null. Rules: - `ctx`, `*ctx`, and `outSize` must be non-null. - `*outSize` receives bytes written during final flush/close. - Always call this after successful `initCompressor()`. ### C Decompression Types ```c struct dContext; struct dData { size_t bufferSize; unsigned int jobs; int headerless; char transform[64]; char entropy[16]; unsigned int blockSize; size_t originalSize; int checksum; int bsVersion; }; ``` `dContext` is opaque. `dData` fields: | Field | Direction | Required | Description | | --- | --- | --- | --- | | `bufferSize` | in | Always | Maximum output bytes accepted in one `decompress()` call. Must not exceed 2 GiB. | | `jobs` | in | Always | Maximum concurrent jobs. | | `headerless` | in | Always | Non-zero means the compressed stream has no header. | | `transform` | in/out | Headerless | Transform name or chain. Validated and normalized in headerless mode. | | `entropy` | in/out | Headerless | Entropy codec name. Validated and normalized in headerless mode. | | `blockSize` | in/out | Headerless | Original compression block size. Rounded up to a multiple of 16 by `initDecompressor()`. | | `originalSize` | in | Headerless | Original stream size, when known. | | `checksum` | in | Headerless | `0`, `32`, or `64`. Must match compression. | | `bsVersion` | in | Headerless | Bitstream version to decode. | For headered streams, only `bufferSize`, `jobs`, and `headerless = 0` are required. Stream metadata is read from the bitstream. ### `getDecompressorVersion` ```c unsigned int getDecompressorVersion(void); ``` Returns the decompressor C API version packed as: ```text (major << 16) | (minor << 8) | patch ``` Current header constants are `KANZI_DECOMP_VERSION_MAJOR`, `KANZI_DECOMP_VERSION_MINOR`, and `KANZI_DECOMP_VERSION_PATCH`. ### `initDecompressor` ```c int initDecompressor(struct dData* dParam, FILE* src, struct dContext** ctx); ``` Initializes a decompressor reading from `src`. Rules: - `dParam`, `src`, and `ctx` must be non-null. - `bufferSize` must be at most 2 GiB. - In headerless mode, `transform` and `entropy` must be null-terminated inside their fixed arrays. - In headerless mode, `transform` and `entropy` are validated and normalized in place. - On success, `*ctx` receives a new decompressor context. ### `decompress` ```c int decompress(struct dContext* ctx, unsigned char* dst, size_t* inSize, size_t* outSize); ``` Decompresses bytes into `dst`. Rules: - `ctx` and `outSize` must be non-null. - `*outSize` is the destination capacity on input. - `*outSize` must be less than or equal to the initialized `bufferSize`. - If input `*outSize == 0`, the call returns success and writes nothing. - `dst` must be non-null when input `*outSize != 0`. - On success, `*outSize` is replaced with the number of bytes decoded. - If `inSize` is non-null, it receives the number of compressed bytes read during the call. End of stream is reported as a successful call with `*outSize == 0`. ### `disposeDecompressor` ```c int disposeDecompressor(struct dContext** ctx); ``` Closes the decompressor, releases resources, and sets `*ctx` to null. Rules: - `ctx` and `*ctx` must be non-null. - Always call this after successful `initDecompressor()`. ### C Example ```c #include #include #include "api/Compressor.hpp" #include "api/Decompressor.hpp" int write_kanzi_file(const char* path, const unsigned char* data, size_t len) { FILE* out = fopen(path, "wb"); if (out == NULL) return -1; struct cData params; memset(¶ms, 0, sizeof(params)); strcpy(params.transform, "LZ"); strcpy(params.entropy, "ANS0"); params.blockSize = 1 << 20; params.jobs = 1; params.checksum = 32; params.headerless = 0; struct cContext* ctx = NULL; int rc = initCompressor(¶ms, out, &ctx); if (rc != 0) { fclose(out); return rc; } size_t written = 0; rc = compress(ctx, data, len, &written); size_t flushed = 0; int close_rc = disposeCompressor(&ctx, &flushed); fclose(out); return (rc != 0) ? rc : close_rc; } ``` ## C++ Stream API Headers: ```cpp #include "io/CompressedOutputStream.hpp" #include "io/CompressedInputStream.hpp" ``` `kanzi::InputStream` is an alias for `std::istream`. `kanzi::OutputStream` is an alias for `std::ostream`. The compressed streams derive from those standard stream types by using the wrapped stream's `rdbuf()`. The caller owns the underlying stream and must keep it alive for the lifetime of the Kanzi stream. ### `CompressedOutputStream` ```cpp class kanzi::CompressedOutputStream : public kanzi::OutputStream { public: CompressedOutputStream(OutputStream& os, int jobs = 1, const std::string& entropy = "NONE", const std::string& transform = "NONE", int blockSize = 4 * 1024 * 1024, int checksum = 0, uint64 originalSize = 0, #ifdef CONCURRENCY_ENABLED ThreadPool* pool = nullptr, #endif bool headerless = false); CompressedOutputStream(OutputStream& os, Context& ctx, bool headerless = false); ~CompressedOutputStream(); bool addListener(Listener& listener); bool removeListener(Listener& listener); std::ostream& write(const char* data, std::streamsize length); std::ostream& put(char c); std::ostream& flush(); std::streampos tellp(); std::ostream& seekp(std::streampos pos); void close(); uint64 getWritten() const; }; ``` Constructor parameters: | Parameter | Description | | --- | --- | | `os` | Destination stream. Must remain alive. | | `jobs` | Number of concurrent jobs. | | `entropy` | Entropy codec name. | | `transform` | Transform name or chain. | | `blockSize` | Compression block size in bytes. Must be `1024..1 GiB` and a multiple of 16. | | `checksum` | `0`, `32`, or `64`. | | `originalSize` | Optional original input size stored in the header when known. | | `pool` | Optional caller-owned thread pool when concurrency is enabled. | | `headerless` | If true, omit stream header. | Context constructor keys are listed in [Context Keys](#context-keys). Methods: | Method | Description | | --- | --- | | `write(data, length)` | Compresses `length` bytes from `data`. Throws on negative length, closed stream, write failure, or codec failure. | | `put(c)` | Writes one byte. | | `flush()` | No-op for Kanzi buffering. Underlying stream flushing remains caller-controlled. | | `close()` | Finishes all pending blocks, closes the compressed bitstream, and releases internal resources. | | `getWritten()` | Returns compressed bytes written so far. | | `addListener(listener)` | Registers an event listener. | | `removeListener(listener)` | Unregisters an event listener. Returns false if not registered. | | `tellp()` | Not supported. Throws `std::ios_base::failure`. | | `seekp(pos)` | Not supported. Throws `std::ios_base::failure`. | Call `close()` before destroying or reading the generated compressed data. The destructor also cleans up, but explicit close makes error handling deterministic. ### `CompressedInputStream` ```cpp class kanzi::CompressedInputStream : public kanzi::InputStream { public: CompressedInputStream(InputStream& is, int jobs = 1, const std::string& entropy = "NONE", const std::string& transform = "NONE", int blockSize = 4 * 1024 * 1024, int checksum = 0, uint64 originalSize = 0, #ifdef CONCURRENCY_ENABLED ThreadPool* pool = nullptr, #endif bool headerless = false, int bsVersion = BITSTREAM_FORMAT_VERSION); CompressedInputStream(InputStream& is, Context& ctx, bool headerless = false); ~CompressedInputStream(); bool addListener(Listener& listener); bool removeListener(Listener& listener); std::istream& read(char* data, std::streamsize length); std::streamsize gcount() const; int get(); int peek(); void close(); uint64 getRead() const; #if !defined(_MSC_VER) || _MSC_VER > 1500 bool seek(int64 bitPos); int64 tell(); #endif std::streampos tellg(); std::istream& seekg(std::streampos pos); std::istream& putback(char c); std::istream& unget(); }; ``` Constructor behavior: - For headered streams (`headerless == false`), metadata is read from the compressed stream. Constructor `entropy`, `transform`, `blockSize`, `checksum`, and `originalSize` are not authoritative after header decoding. - For headerless streams (`headerless == true`), the caller must provide matching `entropy`, `transform`, `blockSize`, `checksum`, and bitstream version. Methods: | Method | Description | | --- | --- | | `read(data, length)` | Decodes up to `length` bytes into `data`. Sets `gcount()`. Throws on negative length, closed stream, invalid stream, checksum failure, or codec failure. | | `gcount()` | Returns bytes decoded by the last `read()` or `get()`. | | `get()` | Decodes and returns one byte, or `EOF`. | | `peek()` | Returns next byte without consuming it, or `EOF`. | | `close()` | Closes the compressed input stream and releases internal resources. | | `getRead()` | Returns compressed bytes consumed so far. | | `seek(bitPos)` | Seeks to a bit position. Valid positions are block boundaries. Returns false on invalid/closed stream or failed underlying seek. | | `tell()` | Returns current bit position from the underlying bitstream. | | `tellg()` | Not supported. Throws `std::ios_base::failure`. | | `seekg(pos)` | Not supported. Throws `std::ios_base::failure`. | | `putback(c)` | Not supported. Sets `badbit` and throws. | | `unget()` | Not supported. Sets `badbit` and throws. | ### C++ Stream Example ```cpp #include #include "io/CompressedOutputStream.hpp" #include "io/CompressedInputStream.hpp" void round_trip(const char* input_path, const char* compressed_path, const char* output_path) { { std::ifstream in(input_path, std::ios::binary); std::ofstream out(compressed_path, std::ios::binary); kanzi::CompressedOutputStream cos(out, 1, "ANS0", "LZ", 1 << 20, 32); char buffer[65536]; while (in.good()) { in.read(buffer, sizeof(buffer)); std::streamsize n = in.gcount(); if (n > 0) cos.write(buffer, n); } cos.close(); } { std::ifstream in(compressed_path, std::ios::binary); std::ofstream out(output_path, std::ios::binary); kanzi::CompressedInputStream cis(in, 1); char buffer[65536]; while (true) { cis.read(buffer, sizeof(buffer)); std::streamsize n = cis.gcount(); if (n <= 0) break; out.write(buffer, n); } cis.close(); } } ``` ## Context Keys `kanzi::Context` is a string-keyed parameter map used by stream constructors, factories, transforms, and entropy codecs. ```cpp class kanzi::Context { public: bool has(const std::string& key) const; int getInt(const std::string& key, int defValue = 0) const; int64 getLong(const std::string& key, int64 defValue = 0) const; std::string getString(const std::string& key, const std::string& defValue = "") const; void putInt(const std::string& key, int value); void putLong(const std::string& key, int64 value); void putString(const std::string& key, const std::string& value); #ifdef CONCURRENCY_ENABLED ThreadPool* getPool() const; #endif }; ``` Common stream keys: | Key | Type | Used by | Description | | --- | --- | --- | --- | | `jobs` | int | Input/output streams, BWT, codecs | Concurrent jobs. | | `blockSize` | int | Input/output streams, transforms, entropy predictors | Block size in bytes. | | `checksum` | int | Input/output streams | `0`, `32`, or `64`. | | `entropy` | string | Input/output streams, factories, transforms | Entropy codec name. | | `transform` | string | Input/output streams, factories, transforms | Transform name or chain. | | `bsVersion` | int | Input stream, output stream context, version-sensitive codecs | Bitstream version. Defaults to current version in headerless input mode when omitted. | | `outputSize` | int64 | Headerless input stream | Optional original decoded size. | | `size` | int | Some entropy predictors | Current block size hint. | | `dataType` | int | Transforms | Internal detected data type passed between transforms. | | `textcodec` | int | `TEXT` transform | Internal text codec selection. | | `packOnlyDNA` | int | `DNA` transform | Internal flag for DNA-only packing. | Only the stream keys should be considered stable integration inputs. Other keys are implementation details used by codec pipelines. Context constructor requirements: ```cpp kanzi::Context ctx; ctx.putInt("jobs", 1); ctx.putString("entropy", "ANS0"); ctx.putString("transform", "LZ"); ctx.putInt("blockSize", 1 << 20); ctx.putInt("checksum", 32); kanzi::CompressedOutputStream cos(out, ctx); ``` For headerless input: ```cpp kanzi::Context ctx; ctx.putInt("jobs", 1); ctx.putString("entropy", "ANS0"); ctx.putString("transform", "LZ"); ctx.putInt("blockSize", 1 << 20); ctx.putInt("checksum", 32); ctx.putInt("bsVersion", 6); // optional; defaults to current version ctx.putLong("outputSize", size); // optional kanzi::CompressedInputStream cis(in, ctx, true); ``` ## Transforms Header: ```cpp #include "Transform.hpp" #include "transform/TransformFactory.hpp" ``` ### `Transform` ```cpp template class kanzi::Transform { public: virtual bool forward(SliceArray& src, SliceArray& dst, int length) = 0; virtual bool inverse(SliceArray& src, SliceArray& dst, int length) = 0; virtual int getMaxEncodedLength(int srcLen) const = 0; virtual ~Transform(); }; ``` Contract: - `forward()` transforms `length` elements from `src` into `dst`. - `inverse()` reverses `forward()`. - Both methods update `SliceArray::_index` on success according to consumed/produced elements. - Return `false` when the transform elects not to apply or cannot encode/decode the provided data. - Throw `std::invalid_argument` for invalid slice arguments. - Implementations must not retain cross-call state that changes output depending on prior blocks. ### `SliceArray` ```cpp template class kanzi::SliceArray { public: T* _array; int _length; int _index; SliceArray(T* arr, int len, int index = 0); static bool isValid(const SliceArray& sa); }; ``` `_length` is the total buffer capacity. `_index` is the current offset. `SliceArray` does not own the buffer memory. ### `TransformFactory` ```cpp template class kanzi::TransformFactory { public: static uint64 getType(const char* name); static uint64 getTypeToken(const char* name); static std::string getName(uint64 functionType); static TransformSequence* newTransform(Context& ctx, uint64 functionType); }; ``` Methods: | Method | Description | | --- | --- | | `getType(name)` | Parses a transform chain such as `BWT+RANK+ZRLT` into an encoded `uint64`. Throws on unknown transform or more than 8 transforms. | | `getTypeToken(name)` | Parses a single transform token into its type id. | | `getName(functionType)` | Converts an encoded transform chain back to canonical names. | | `newTransform(ctx, functionType)` | Allocates a `TransformSequence` for the encoded chain. Caller owns the returned pointer. | ### `TransformSequence` Header: ```cpp #include "transform/TransformSequence.hpp" ``` `TransformSequence` is itself a `Transform` that applies up to 8 transforms in order for `forward()` and in reverse order for `inverse()`. When constructed with `deallocate = true`, it owns and deletes the transform pointers. ## Entropy Codecs Headers: ```cpp #include "EntropyEncoder.hpp" #include "EntropyDecoder.hpp" #include "entropy/EntropyEncoderFactory.hpp" #include "entropy/EntropyDecoderFactory.hpp" ``` ### `EntropyEncoder` ```cpp class kanzi::EntropyEncoder { public: virtual int encode(const byte block[], uint blkptr, uint len) = 0; virtual OutputBitStream& getBitStream() const = 0; virtual void dispose() = 0; virtual ~EntropyEncoder(); }; ``` `encode()` writes `len` bytes from `block + blkptr` and returns the number of bytes processed. Call `dispose()` before destroying or abandoning an encoder. ### `EntropyDecoder` ```cpp class kanzi::EntropyDecoder { public: virtual int decode(byte block[], uint blkptr, uint len) = 0; virtual InputBitStream& getBitStream() const = 0; virtual void dispose() = 0; virtual ~EntropyDecoder(); }; ``` `decode()` writes `len` decoded bytes to `block + blkptr` and returns the number of bytes processed. Call `dispose()` before destroying or abandoning a decoder. ### Entropy Factories ```cpp class kanzi::EntropyEncoderFactory { public: static EntropyEncoder* newEncoder(OutputBitStream& obs, Context& ctx, short entropyType); static const char* getName(short entropyType); static short getType(const char* name); }; class kanzi::EntropyDecoderFactory { public: static EntropyDecoder* newDecoder(InputBitStream& ibs, Context& ctx, short entropyType); static const char* getName(short entropyType); static short getType(const char* name); }; ``` Caller owns codecs returned by `newEncoder()` and `newDecoder()`. Factory type constants: | Constant | Name | | --- | --- | | `NONE_TYPE` | `NONE` | | `HUFFMAN_TYPE` | `HUFFMAN` | | `FPAQ_TYPE` | `FPAQ` | | `RANGE_TYPE` | `RANGE` | | `ANS0_TYPE` | `ANS0` | | `ANS1_TYPE` | `ANS1` | | `CM_TYPE` | `CM` | | `TPAQ_TYPE` | `TPAQ` | | `TPAQX_TYPE` | `TPAQX` | ## Bitstream API Headers: ```cpp #include "InputBitStream.hpp" #include "OutputBitStream.hpp" #include "bitstream/DefaultInputBitStream.hpp" #include "bitstream/DefaultOutputBitStream.hpp" ``` ### `InputBitStream` ```cpp class kanzi::InputBitStream { public: virtual int readBit() = 0; virtual uint64 readBits(uint length) = 0; virtual uint readBits(byte bits[], uint length) = 0; virtual void close() = 0; virtual uint64 read() const = 0; virtual bool hasMoreToRead() = 0; virtual ~InputBitStream(); }; ``` `readBits(uint length)` accepts lengths in `[1..64]`. ### `OutputBitStream` ```cpp class kanzi::OutputBitStream { public: virtual void writeBit(int bit) = 0; virtual uint writeBits(uint64 bits, uint length) = 0; virtual uint writeBits(const byte bits[], uint length) = 0; virtual void close() = 0; virtual uint64 written() const = 0; virtual ~OutputBitStream(); }; ``` `writeBits(uint64 bits, uint length)` accepts lengths in `[1..64]`; invalid lengths return 0 in the default implementation. ### Default Bitstreams ```cpp class kanzi::DefaultInputBitStream : public InputBitStream, public Seekable { public: DefaultInputBitStream(InputStream& is, uint bufferSize = 65536); int readBit(); uint64 readBits(uint length); uint readBits(byte bits[], uint count); void close(); uint64 read() const; bool hasMoreToRead(); bool isClosed() const; int64 tell(); bool seek(int64 pos); }; class kanzi::DefaultOutputBitStream : public OutputBitStream, public Seekable { public: DefaultOutputBitStream(OutputStream& os, uint bufferSize = 65536); void writeBit(int bit); uint writeBits(uint64 bits, uint length); uint writeBits(const byte bits[], uint length); void close(); uint64 written() const; bool isClosed() const; int64 tell(); bool seek(int64 pos); }; ``` Default bitstream buffer sizes must be: - At least 1024 bytes - At most 536,870,912 bytes - A multiple of 8 `DefaultInputBitStream::seek()` accepts any non-negative bit position and consumes partial-byte bits after seeking. `DefaultOutputBitStream::seek()` only accepts byte-aligned bit positions. ## Events and Listeners Headers: ```cpp #include "Event.hpp" #include "Listener.hpp" ``` Streams can notify caller-provided listeners during compression and decompression. ### `Listener` ```cpp template class kanzi::Listener { public: virtual void processEvent(const T& evt) = 0; virtual ~Listener(); }; ``` Register with: ```cpp cos.addListener(listener); cis.addListener(listener); ``` ### `Event` ```cpp class kanzi::Event { public: enum Type { COMPRESSION_START, COMPRESSION_END, BEFORE_TRANSFORM, AFTER_TRANSFORM, BEFORE_ENTROPY, AFTER_ENTROPY, DECOMPRESSION_START, DECOMPRESSION_END, AFTER_HEADER_DECODING, BLOCK_INFO }; enum HashType { NO_HASH, SIZE_32, SIZE_64 }; struct HeaderInfo { std::string inputName; int bsVersion; int checksumSize; int blockSize; std::string entropyType; std::string transformType; int64 originalSize; int64 fileSize; }; int getId() const; int64 getSize() const; Type getType() const; WallTimer::TimeData getTime() const; uint64 getHash() const; int64 getOffset() const; HashType getHashType() const; HeaderInfo* getInfo() const; std::string toString() const; std::string getTypeAsString() const; }; ``` Common event data: | Getter | Meaning | | --- | --- | | `getType()` | Event kind. | | `getId()` | Block id or event id, depending on type. | | `getSize()` | Block size or stage size. | | `getHash()` | Block checksum when enabled, otherwise 0. | | `getHashType()` | No hash, 32-bit hash, or 64-bit hash. | | `getOffset()` | Compressed block offset when available. | | `getInfo()` | Header metadata for `AFTER_HEADER_DECODING`. | Minimal listener: ```cpp class MyListener : public kanzi::Listener { public: void processEvent(const kanzi::Event& evt) { std::cout << evt.toString() << std::endl; } }; ``` ## Errors and Exceptions ### C Error Codes `src/Error.hpp` defines: | Code | Name | | --- | --- | | 1 | `ERR_MISSING_PARAM` | | 2 | `ERR_BLOCK_SIZE` | | 3 | `ERR_INVALID_CODEC` | | 4 | `ERR_CREATE_COMPRESSOR` | | 5 | `ERR_CREATE_DECOMPRESSOR` | | 6 | `ERR_OUTPUT_IS_DIR` | | 7 | `ERR_OVERWRITE_FILE` | | 8 | `ERR_CREATE_FILE` | | 9 | `ERR_CREATE_BITSTREAM` | | 10 | `ERR_OPEN_FILE` | | 11 | `ERR_READ_FILE` | | 12 | `ERR_WRITE_FILE` | | 13 | `ERR_PROCESS_BLOCK` | | 14 | `ERR_CREATE_CODEC` | | 15 | `ERR_INVALID_FILE` | | 16 | `ERR_STREAM_VERSION` | | 17 | `ERR_CREATE_STREAM` | | 18 | `ERR_INVALID_PARAM` | | 19 | `ERR_CRC_CHECK` | | 20 | `ERR_RESERVED_NAME` | | 127 | `ERR_UNKNOWN` | ### `IOException` Header: ```cpp #include "io/IOException.hpp" ``` ```cpp class kanzi::IOException : public std::runtime_error { public: IOException(const std::string& msg); IOException(const std::string& msg, int error); int error() const; }; ``` `error()` returns a value from `kanzi::Error::ErrorCode`. ### `BitStreamException` Header: ```cpp #include "BitStreamException.hpp" ``` ```cpp class kanzi::BitStreamException : public std::runtime_error { public: enum BitStreamStatus { UNDEFINED = 0, INPUT_OUTPUT = 1, END_OF_STREAM = 2, INVALID_STREAM = 3, STREAM_CLOSED = 4 }; BitStreamException(const std::string& msg); BitStreamException(const std::string& msg, int code); int error() const; }; ``` ## Python Wrapper Files: - `src/api/kanzi_c_api.py`: low-level `ctypes` declarations - `src/api/kanzi.py`: higher-level Python classes The wrapper expects the Kanzi shared library to be loadable by name: | Platform | Library name | | --- | --- | | Windows | `kanzi.dll` | | macOS | `libkanzi.dylib` | | Linux/Unix | `libkanzi.so` | Ensure the shared library is in the dynamic loader path before importing. ### `KanziError` ```python class KanziError(RuntimeError): pass ``` Raised when a wrapped C function returns a non-zero Kanzi error code. ### `Compressor` ```python class Compressor: def __init__( self, dst_path, transform=b"LZ", entropy=b"Huffman", block_size=1 << 20, jobs=1, checksum=0, headerless=0, ) def compress(self, data: bytes) -> int def close(self) -> int ``` Behavior: - Opens `dst_path` for binary writing. - Initializes the C compressor. - `compress(data)` writes one block and returns bytes written by that call. - `close()` disposes the C compressor, closes the file, and returns bytes flushed during close. - Supports context-manager use. Example: ```python from kanzi import Compressor with Compressor("sample.knz", transform=b"LZ", entropy=b"ANS0", checksum=32) as c: c.compress(b"hello kanzi") ``` ### `Decompressor` ```python class Decompressor: def __init__( self, src_path, buffer_size, jobs=1, headerless=0, **headerless_params, ) def decompress_block(self, max_output: int) -> bytes def close(self) ``` Behavior: - Opens `src_path` for binary reading. - Initializes the C decompressor. - `decompress_block(max_output)` returns up to `max_output` decoded bytes. - An empty result indicates end of stream. - Supports context-manager use. Headerless parameters, when `headerless=1`: | Python keyword | C field | | --- | --- | | `transform` | `dData.transform` | | `entropy` | `dData.entropy` | | `blockSize` | `dData.blockSize` | | `originalSize` | `dData.originalSize` | | `checksum` | `dData.checksum` | | `bsVersion` | `dData.bsVersion` | Example: ```python from kanzi import Decompressor out = bytearray() with Decompressor("sample.knz", buffer_size=1 << 20) as d: while True: block = d.decompress_block(1 << 20) if not block: break out.extend(block) ``` ## Integration Notes - Prefer the C API for ABI-stable dynamic-library integration. - Prefer the C++ stream API when embedding directly in C++ applications. - Keep the underlying `FILE*`, `std::istream`, or `std::ostream` open while the Kanzi context/stream exists. - Always dispose or close compressors to flush final bytes. - Use headered streams unless another protocol stores Kanzi parameters. - Match headerless decompression parameters exactly to compression parameters. - Do not share one stream instance concurrently between threads. - Transform and entropy objects returned by factories are heap-allocated; delete them when no longer needed unless ownership is transferred to `TransformSequence`. - Check return codes in the C API and catch `kanzi::IOException`, `kanzi::BitStreamException`, `std::invalid_argument`, and `std::ios_base::failure` in C++ integrations. kanzi-cpp-2.6.0/CMakeLists.txt000066400000000000000000000326771525576310500162030ustar00rootroot00000000000000cmake_minimum_required(VERSION 3.10) set(KANZI_PROJECT_VERSION "2.6.0") set(KANZI_ABI_VERSION "1.0.0") project(kanzi VERSION ${KANZI_PROJECT_VERSION} LANGUAGES C CXX) include(GNUInstallDirs) include(CMakePackageConfigHelpers) string(REPLACE "." ";" KANZI_ABI_VERSION_PARTS "${KANZI_ABI_VERSION}") list(GET KANZI_ABI_VERSION_PARTS 0 KANZI_SOVERSION) option(KANZI_ENABLE_NATIVE_OPTIMIZATIONS "Enable CPU-specific optimizations such as -march=native" ON) option(KANZI_USE_INTRINSICS "Enable platform-specific intrinsic implementations" OFF) set(KANZI_INSTALL_RELATIVE_RPATH_DEFAULT ON) if(CMAKE_INSTALL_PREFIX STREQUAL "/usr") set(KANZI_INSTALL_RELATIVE_RPATH_DEFAULT OFF) endif() option(KANZI_INSTALL_RELATIVE_RPATH "Install the CLI with a relative runtime search path for the shared library" ${KANZI_INSTALL_RELATIVE_RPATH_DEFAULT} ) set(KANZI_CMAKE_INSTALL_CMAKEDIR "${CMAKE_INSTALL_LIBDIR}/cmake/Kanzi") # Set C++ standard (user-configurable, defaults to C++17) if(NOT DEFINED CMAKE_CXX_STANDARD) set(CMAKE_CXX_STANDARD 17) endif() set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) #set(CMAKE_CXX_COMPILER "clang++") # Set C standard (for TestAPI.c) set(CMAKE_C_STANDARD 11) #set(CMAKE_C_STANDARD_REQUIRED True) #set(CMAKE_C_COMPILER "clang") set(CMAKE_THREAD_PREFER_PTHREAD TRUE) find_package(Threads REQUIRED) # ---------------------------------------------- if(CONCURRENCY_DISABLED) add_definitions(-DCONCURRENCY_DISABLED) endif() if(NOT WIN32) add_definitions(-D_FILE_OFFSET_BITS=64) endif() if(MSVC) set(COMMON_FLAGS "/W4 /O2 /DNDEBUG") if(KANZI_USE_INTRINSICS) string(APPEND COMMON_FLAGS " /DUSE_INTRINSICS") endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS} /GR-") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}") else() set(COMMON_FLAGS "-Wall -Wextra -O3 -fomit-frame-pointer -fPIC -DNDEBUG -pedantic") if(KANZI_USE_INTRINSICS) string(APPEND COMMON_FLAGS " -DUSE_INTRINSICS") endif() if(KANZI_ENABLE_NATIVE_OPTIMIZATIONS) string(APPEND COMMON_FLAGS " -march=native") endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON_FLAGS} -fno-rtti") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON_FLAGS}") endif() set(SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src") # Source files set(LIB_COMMON_SOURCES ${SRC_DIR}/Global.cpp ${SRC_DIR}/Event.cpp ${SRC_DIR}/util/WallTimer.cpp ${SRC_DIR}/entropy/EntropyUtils.cpp ${SRC_DIR}/entropy/HuffmanCommon.cpp ${SRC_DIR}/entropy/CMPredictor.cpp ${SRC_DIR}/entropy/TPAQPredictor.cpp ${SRC_DIR}/transform/AliasCodec.cpp ${SRC_DIR}/transform/BWT.cpp ${SRC_DIR}/transform/BWTS.cpp ${SRC_DIR}/transform/DivSufSort.cpp ${SRC_DIR}/transform/SBRT.cpp ${SRC_DIR}/transform/BWTBlockCodec.cpp ${SRC_DIR}/transform/LZCodec.cpp ${SRC_DIR}/transform/FSDCodec.cpp ${SRC_DIR}/transform/ROLZCodec.cpp ${SRC_DIR}/transform/RLT.cpp ${SRC_DIR}/transform/SRT.cpp ${SRC_DIR}/transform/TextCodec.cpp ${SRC_DIR}/transform/UTFCodec.cpp ${SRC_DIR}/transform/EXECodec.cpp ${SRC_DIR}/transform/ZRLT.cpp ) set(LIB_COMP_SOURCES ${SRC_DIR}/api/Compressor.cpp ${SRC_DIR}/bitstream/DebugOutputBitStream.cpp ${SRC_DIR}/bitstream/DefaultOutputBitStream.cpp ${SRC_DIR}/io/CompressedOutputStream.cpp ${SRC_DIR}/entropy/ANSRangeEncoder.cpp ${SRC_DIR}/entropy/BinaryEntropyEncoder.cpp ${SRC_DIR}/entropy/ExpGolombEncoder.cpp ${SRC_DIR}/entropy/FPAQEncoder.cpp ${SRC_DIR}/entropy/HuffmanEncoder.cpp ${SRC_DIR}/entropy/RangeEncoder.cpp ) set(LIB_DECOMP_SOURCES ${SRC_DIR}/api/Decompressor.cpp ${SRC_DIR}/bitstream/DebugInputBitStream.cpp ${SRC_DIR}/bitstream/DefaultInputBitStream.cpp ${SRC_DIR}/io/CompressedInputStream.cpp ${SRC_DIR}/entropy/ANSRangeDecoder.cpp ${SRC_DIR}/entropy/BinaryEntropyDecoder.cpp ${SRC_DIR}/entropy/ExpGolombDecoder.cpp ${SRC_DIR}/entropy/FPAQDecoder.cpp ${SRC_DIR}/entropy/HuffmanDecoder.cpp ${SRC_DIR}/entropy/RangeDecoder.cpp ) set(TEST_SOURCES ${SRC_DIR}/test/TestEntropyCodec.cpp ${SRC_DIR}/test/TestBWT.cpp ${SRC_DIR}/test/TestCompressedStream.cpp ${SRC_DIR}/test/TestDefaultBitStream.cpp ${SRC_DIR}/test/TestTransforms.cpp ${SRC_DIR}/test/TestAPI.cpp ) set(APP_SOURCES ${SRC_DIR}/app/Kanzi.cpp ${SRC_DIR}/app/InfoPrinter.cpp ${SRC_DIR}/app/BlockCompressor.cpp ${SRC_DIR}/app/BlockDecompressor.cpp ) # Libraries add_library(libkanzi STATIC ${LIB_COMMON_SOURCES} ${LIB_COMP_SOURCES} ${LIB_DECOMP_SOURCES}) add_library(libkanzi_shared SHARED ${LIB_COMMON_SOURCES} ${LIB_COMP_SOURCES} ${LIB_DECOMP_SOURCES}) # This ensures -lpthread or -pthread is added to any executable linking these libs target_link_libraries(libkanzi PUBLIC Threads::Threads) target_link_libraries(libkanzi_shared PUBLIC Threads::Threads) target_include_directories(libkanzi PUBLIC $ $ ) target_include_directories(libkanzi_shared PUBLIC $ $ ) set_target_properties(libkanzi PROPERTIES OUTPUT_NAME "kanzi") set_target_properties(libkanzi_shared PROPERTIES OUTPUT_NAME "kanzi" VERSION "${KANZI_ABI_VERSION}" SOVERSION "${KANZI_SOVERSION}" ) #add_library(libkanzi_comp STATIC ${LIB_COMP_SOURCES}) #add_library(libkanzi_decomp STATIC ${LIB_DECOMP_SOURCES}) #add_library(libkanzi_comp_shared SHARED ${LIB_COMP_SOURCES}) #add_library(libkanzi_decomp_shared SHARED ${LIB_DECOMP_SOURCES}) # Executable target for C++ add_executable(testBWT ${SRC_DIR}/test/TestBWT.cpp) target_link_libraries(testBWT libkanzi) add_executable(testTransforms ${SRC_DIR}/test/TestTransforms.cpp) target_link_libraries(testTransforms libkanzi) add_executable(testEntropyCodec ${SRC_DIR}/test/TestEntropyCodec.cpp) target_link_libraries(testEntropyCodec libkanzi) add_executable(testDefaultBitStream ${SRC_DIR}/test/TestDefaultBitStream.cpp) target_link_libraries(testDefaultBitStream libkanzi) add_executable(testFactories ${SRC_DIR}/test/TestFactories.cpp) target_link_libraries(testFactories libkanzi) add_executable(testMalformedStream ${SRC_DIR}/test/TestMalformedStream.cpp) target_link_libraries(testMalformedStream libkanzi) add_executable(testCompressedStream ${SRC_DIR}/test/TestCompressedStream.cpp) target_link_libraries(testCompressedStream libkanzi) # Executable target for C API test (TestAPI.c) add_executable(testAPI ${SRC_DIR}/test/TestAPI.c) target_link_libraries(testAPI libkanzi) # IMPORTANT: Force use of C++ Linker because we link against C++ lib 'archon_static' set_target_properties(testAPI PROPERTIES LINKER_LANGUAGE CXX) # Main executable # Dynamically linked executable add_executable(kanzi ${APP_SOURCES}) target_link_libraries(kanzi libkanzi_shared) # Keep the CLI runnable from relocatable prefixes, but avoid embedding an # RPATH for standard system package installs. if(UNIX AND NOT APPLE AND KANZI_INSTALL_RELATIVE_RPATH) set_target_properties(kanzi PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR}" ) endif() # Statically linked executable add_executable(kanzi_static ${APP_SOURCES}) target_link_libraries(kanzi_static libkanzi) # Custom target to build all tests (Named to avoid conflict with CTest 'make test') add_custom_target(build_tests DEPENDS testBWT testTransforms testEntropyCodec testDefaultBitStream testFactories testMalformedStream testCompressedStream testAPI ) # --- CTest Configuration --- enable_testing() if(DEFINED ENV{TMPDIR}) set(SYSTEM_TEMP_DIR "$ENV{TMPDIR}") # Linux/macOS standard elseif(DEFINED ENV{TEMP}) set(SYSTEM_TEMP_DIR "$ENV{TEMP}") # Windows standard elseif(DEFINED ENV{TMP}) set(SYSTEM_TEMP_DIR "$ENV{TMP}") # Windows fallback else() set(SYSTEM_TEMP_DIR "/tmp") # Fallback default endif() file(TO_CMAKE_PATH "${SYSTEM_TEMP_DIR}" SYSTEM_TEMP_DIR) # Register executables as CTest tests # Syntax: add_test(NAME COMMAND ) add_test(NAME BWT COMMAND testBWT -noperf) add_test(NAME Transforms COMMAND testTransforms -type=all -noperf) add_test(NAME EntropyCodec COMMAND testEntropyCodec -type=all -noperf) add_test(NAME DefaultBitStream COMMAND testDefaultBitStream ${SYSTEM_TEMP_DIR}/testDefaultBitStream.tmp -noperf) add_test(NAME Factories COMMAND testFactories) add_test(NAME MalformedStream COMMAND testMalformedStream) add_test(NAME CompressedStream COMMAND testCompressedStream) add_test(NAME API COMMAND testAPI) # Custom target to build static libraries add_custom_target(static_lib DEPENDS libkanzi #libkanzi_comp libkanzi_decomp ) # Custom target to build shared libraries add_custom_target(shared_lib DEPENDS libkanzi_shared #libkanzi_comp_shared libkanzi_decomp_shared ) # Custom target to build all libraries (static and shared) add_custom_target(lib DEPENDS static_lib shared_lib ) # Install the statically linked executable install(TARGETS kanzi_static RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" COMPONENT Runtime ) # Install dynamically linked executable install(TARGETS kanzi RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" COMPONENT Runtime ) # Install the man page (prefer pre-compressed source file). set(KANZI_MANPAGE_GZ "${CMAKE_CURRENT_SOURCE_DIR}/doc/kanzi.1.gz") set(KANZI_MANPAGE "${CMAKE_CURRENT_SOURCE_DIR}/doc/kanzi.1") set(KANZI_MANPAGE_TO_INSTALL "") if(EXISTS "${KANZI_MANPAGE_GZ}") set(KANZI_MANPAGE_TO_INSTALL "${KANZI_MANPAGE_GZ}") elseif(EXISTS "${KANZI_MANPAGE}") find_program(GZIP_EXECUTABLE gzip) if(GZIP_EXECUTABLE) set(KANZI_MANPAGE_GZ_BUILD "${CMAKE_CURRENT_BINARY_DIR}/kanzi.1.gz") execute_process( COMMAND "${GZIP_EXECUTABLE}" -n -c "${KANZI_MANPAGE}" OUTPUT_FILE "${KANZI_MANPAGE_GZ_BUILD}" RESULT_VARIABLE KANZI_GZIP_RESULT ) if(NOT KANZI_GZIP_RESULT EQUAL 0) message(FATAL_ERROR "Failed to gzip man page: ${KANZI_MANPAGE}") endif() set(KANZI_MANPAGE_TO_INSTALL "${KANZI_MANPAGE_GZ_BUILD}") else() message(WARNING "gzip not found. Installing uncompressed man page.") set(KANZI_MANPAGE_TO_INSTALL "${KANZI_MANPAGE}") endif() endif() if(KANZI_MANPAGE_TO_INSTALL) install(FILES "${KANZI_MANPAGE_TO_INSTALL}" DESTINATION "${CMAKE_INSTALL_MANDIR}/man1" COMPONENT Runtime ) endif() # Install the libraries (static and shared) install(TARGETS libkanzi libkanzi_shared EXPORT KanziTargets ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT Development LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}" COMPONENT Runtime NAMELINK_COMPONENT Development RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" COMPONENT Runtime INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/kanzi" ) # Dynamically discover and install public headers # We only install headers from the library tree, preserving paths relative to # src/ so includes like and # remain stable across platforms. set(PUBLIC_HEADER_BASE_DIRS ${SRC_DIR} ${SRC_DIR}/api ${SRC_DIR}/bitstream ${SRC_DIR}/entropy ${SRC_DIR}/io ${SRC_DIR}/transform ${SRC_DIR}/util ) # List to hold all found header files set(ALL_PUBLIC_HEADERS) # Iterate through the base directories and find headers foreach(dir IN LISTS PUBLIC_HEADER_BASE_DIRS) if(EXISTS "${dir}") file(GLOB_RECURSE CURRENT_DIR_HEADERS "${dir}/*.h" "${dir}/*.hpp") list(APPEND ALL_PUBLIC_HEADERS ${CURRENT_DIR_HEADERS}) endif() endforeach() list(REMOVE_DUPLICATES ALL_PUBLIC_HEADERS) # Install each header, preserving its relative path within the 'kanzi' include directory foreach(header_file IN LISTS ALL_PUBLIC_HEADERS) file(RELATIVE_PATH REL_PATH "${SRC_DIR}" "${header_file}") if(NOT REL_PATH MATCHES "^(app|test|obj|build)/") get_filename_component(REL_DIR "${REL_PATH}" DIRECTORY) if(REL_DIR STREQUAL ".") set(HEADER_DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/kanzi") else() set(HEADER_DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/kanzi/${REL_DIR}") endif() install(FILES "${header_file}" DESTINATION "${HEADER_DESTINATION}" COMPONENT Development ) endif() endforeach() if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE") install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE" DESTINATION "${CMAKE_INSTALL_DOCDIR}" COMPONENT Runtime ) endif() configure_package_config_file( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/KanziConfig.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/KanziConfig.cmake" INSTALL_DESTINATION "${KANZI_CMAKE_INSTALL_CMAKEDIR}" ) write_basic_package_version_file( "${CMAKE_CURRENT_BINARY_DIR}/KanziConfigVersion.cmake" VERSION "${PROJECT_VERSION}" COMPATIBILITY SameMajorVersion ) install(EXPORT KanziTargets FILE KanziTargets.cmake NAMESPACE kanzi:: DESTINATION "${KANZI_CMAKE_INSTALL_CMAKEDIR}" COMPONENT Development ) install(FILES "${CMAKE_CURRENT_BINARY_DIR}/KanziConfig.cmake" "${CMAKE_CURRENT_BINARY_DIR}/KanziConfigVersion.cmake" DESTINATION "${KANZI_CMAKE_INSTALL_CMAKEDIR}" COMPONENT Development ) # Uninstall all files added during install configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/cmake_uninstall.cmake.in" "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" @ONLY) add_custom_target(uninstall COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake COMMENT "Uninstalling..." ) kanzi-cpp-2.6.0/LICENSE000066400000000000000000000261221525576310500144340ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2022 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. kanzi-cpp-2.6.0/README.md000066400000000000000000000271731525576310500147150ustar00rootroot00000000000000# Kanzi Kanzi is a modern, modular, portable, and efficient lossless data compressor written in C++. * Modern: Kanzi implements state-of-the-art compression algorithms and is built to fully utilize multi-core CPUs via built-in multi-threading. * Modular: Entropy codecs and data transforms can be selected and combined at runtime to best suit the specific data being compressed. * Portable: Supports a wide range of operating systems, compilers, and C++ standards (details below). * Expandable: A clean, interface-driven design—with no external dependencies—makes Kanzi easy to integrate, extend, and customize. * Efficient: Carefully optimized to balance compression ratio and speed for practical, high-performance usage. Unlike most mainstream lossless compressors, Kanzi is not limited to a single compression paradigm. By combining multiple algorithms and techniques, it supports a broader range of compression ratios and adapts better to diverse data types. Most traditional compressors underutilize modern hardware by running single-threaded—even on machines with many cores. Kanzi, in contrast, is concurrent by design, compressing multiple blocks in parallel across threads for significant performance gains. However, it is not compatible with standard compression formats. It’s important to note that Kanzi is a data compressor, not an archiver. It includes optional checksums for verifying data integrity, but does not provide features like cross-file deduplication or data recovery mechanisms. That said, it produces a seekable bitstream, meaning one or more consecutive blocks can be decompressed independently, without needing to process the entire stream. For more details, see [Wiki](https://github.com/flanglet/kanzi-cpp/wiki), [Q&A](https://github.com/flanglet/kanzi-cpp/wiki/q&a) and [DeepWiki](https://deepwiki.com/flanglet/kanzi-cpp/1-overview) See how to reuse the C and C++ APIs: [here](https://github.com/flanglet/kanzi-cpp/wiki/Using-and-extending-the-code) There is a Java implementation available here: https://github.com/flanglet/kanzi There is a Go implementation available here: https://github.com/flanglet/kanzi-go ![Build Status](https://github.com/flanglet/kanzi-cpp/actions/workflows/c-cpp.yml/badge.svg) [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=flanglet_kanzi-cpp&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=flanglet_kanzi-cpp) [![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=flanglet_kanzi-cpp&metric=ncloc)](https://sonarcloud.io/summary/new_code?id=flanglet_kanzi-cpp) Coverity Scan Build Status [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/flanglet/kanzi-cpp) ## Why Kanzi While excellent open-source compressors like zstd and brotli exist, they are primarily based on Lempel-Ziv (LZ) algorithms. Zstd, in particular, is a fantastic general-purpose choice known for its speed. However, LZ-based tools have inherent limits regarding compression ratios. Kanzi offers a compelling alternative for specific high-performance scenarios: * Beyond LZ: By incorporating Burrows-Wheeler Transform (BWT) and Context Modeling (CM), Kanzi can achieve compression ratios that traditional LZ methods cannot. * Speed where it counts: While LZ is ideal for "compress once, decompress often" (like software distribution), it often slows down significantly at high compression settings. Kanzi leverages multi-core CPUs to maintain performance, making it highly effective for backups, real-time data generation, and one-off transfers. * Content-Aware: Kanzi features built-in, customizable transforms for specific data types (e.g., multimedia, DNA, UTF text), improving efficiency where generic compressors fail. * Extensible: The architecture is developer-friendly, making it straightforward to implement new transforms or entropy codecs for experimentation or niche data types. ## Benchmarks Kanzi version 2.5.0 C++ implementation _Note: The default block size at level 9 is 32MB. This limits the number of threads in use, especially with smaller files like enwik8, but all tests below are performed with default values._ ### silesia.tar Test machine: AMD Ryzen 9 9950X 16-Core Processor running Ubuntu 25.10 Download at http://sun.aei.polsl.pl/~sdeor/corpus/silesia.zip | Compressor | Encoding (ms) | Decoding (ms) | Size | |---------------------------------|-----------------|-----------------|------------------| |Original | | | 211,957,760 | |lz4 1.1.10 -T16 -4 | 18 | 13 | 79,910,851 | |**kanzi -l 1** | **72** | **42** | 79,331,051 | |zstd 1.5.8 -T16 -2 | 6 | 11 | 69,443,247 | |**kanzi -l 2** | **64** | **42** | 68,616,621 | |brotli 1.1.0 -2 | 880 | 333 | 68,040,160 | |gzip 1.13 -9 | 10328 | 704 | 67,651,076 | |**kanzi -l 3** | **109** | **58** | 63,966,794 | |zstd 1.5.8 -T16 -5 | 138 | 123 | 62,867,556 | |**kanzi -l 4** | **194** | **102** | 61,183,757 | |zstd 1.5.8 -T16 -9 | 320 | 114 | 59,233,481 | |brotli 1.1.0 -6 | 4039 | 299 | 58,511,709 | |zstd 1.5.8 -T16 -13 | 1820 | 112 | 57,843,283 | |brotli 1.1.0 -9 | 23030 | 293 | 56,407,229 | |bzip2 1.0.8 -9 | 8223 | 3453 | 54,588,597 | |**kanzi -l 5** | **529** | **255** | 53,853,702 | |zstd 1.5.8 -T16 -19 | 11290 | 130 | 52,830,213 | |**kanzi -l 6** | **919** | **532** | 49,472,084 | |xz 5.8.1 -9 | 43611 | 931 | 48,802,580 | |bsc 3.3.11 -T16 | 1201 | 698 | 47,900,848 | |**kanzi -l 7** | **1153** | **888** | 47,330,422 | |bzip3 1.5.1.r3-g428f422 -j 16 | 2348 | 2218 | 47,260,281 | |**kanzi -l 8** | **4473** | **4881** | 42,962,913 | |**kanzi -l 9** | **11618** | **12381** | 41,520,670 | ![Graph for Silesia on AMD Ryzen 9950X](doc/Plot_silesia.png) Round-trip graph for Silesia on AMD Ryzen 9950X (X = compTime + 2*decompTime, Y = comp size) ### enwik8 Test machine: Apple M3 24 GB macOS Sonoma 15.7.3 Download at https://mattmahoney.net/dc/enwik8.zip | Compressor | Encoding (ms) | Decoding (ms) | Size | |-----------------|----------------|----------------|--------------| |Original | | | 100,000,000 | |kanzi -l 1 | 139 | 85 | 42,870,183 | |kanzi -l 2 | 131 | 92 | 37,544,247 | |kanzi -l 3 | 215 | 123 | 32,551,405 | |kanzi -l 4 | 303 | 170 | 29,536,581 | |kanzi -l 5 | 670 | 372 | 26,528,254 | |kanzi -l 6 | 1009 | 727 | 24,076,765 | |kanzi -l 7 | 1607 | 1366 | 22,817,360 | |kanzi -l 8 | 6371 | 6752 | 21,181,992 | |kanzi -l 9 | 8260 | 8760 | 20,035,144 | ![Graph for enwik8 on AMD Ryzen 9950X](doc/Plot_enwik8.png) Round-trip graph for enwik8 on AMD Ryzen 9950X (X = compTime + 2*decompTime, Y = comp size) ### More benchmarks [Comprehensive lzbench benchmarks](https://github.com/flanglet/kanzi-cpp/wiki/Performance) [More round trip scores](https://github.com/flanglet/kanzi-cpp/wiki/Round%E2%80%90trips-scores) ## Build Kanzi * Platforms: Windows (Visual Studio), Linux, macOS, BSD * Dependencies: None. * Portability: Designed for easy porting to other OSs. * Multithreading: Supported by default. ### Visual Studio The Visual Studio solution and project files are in the `msvc` directory. Open the solution corresponding to your Visual Studio version: * Visual Studio 2008: open `msvc/Kanzi_VS2008.sln`. The solution generates a Windows 32-bit binary. Multithreading is not supported with this version. * Visual Studio 2022: open `msvc/Kanzi_VS2022.sln`. The solution generates Windows binaries and a 64-bit library. * Visual Studio 2026: open `msvc/Kanzi_VS2026.sln`. The solution generates Windows binaries and a 64-bit library. Select the desired configuration and platform in Visual Studio, then build the solution. To build from the command line, open a Developer Command Prompt for the corresponding Visual Studio version and run the following from the repository root: ```text msbuild msvc\Kanzi_VS2022.sln /m /p:Configuration=Release /p:Platform=x64 msbuild msvc\Kanzi_VS2026.sln /m /p:Configuration=Release /p:Platform=x64 ``` Use `/p:Platform=Win32` to build the 32-bit target. For Visual Studio 2008, use the Visual Studio command-line executable instead: ```text VCExpress.exe msvc\Kanzi_VS2008.sln /Build "Debug|Win32" ``` ### mingw-w64 Go to the source directory and run 'make clean && mingw32-make.exe kanzi'. The Makefile contains all the necessary targets. Tested successfully on Win64 with mingw-w64 g++ 8.1.0. Multithreading is supported with g++ version 5.0.0 or newer. Builds successfully with C++11, C++14, C++17. ### Linux Go to the source directory and run 'make clean && make kanzi'. The Makefile contains all the necessary targets. Build successfully on Ubuntu with many versions of g++ and clang++. Multithreading is supported with g++ version 5.0.0 or newer. Builds successfully with C++98, C++11, C++14, C++17, C++20. ### macOS Go to the source directory and run 'make clean && make kanzi'. The Makefile contains all the necessary targets. Build successfully on MacOs with several versions of clang++. Builds successfully with C++98, C++11, C++14, C++17, C++20. ### BSD The makefile uses the gnu-make syntax. First, make sure gmake is present (or install it: 'pkg install gmake'). Go to the source directory and run 'gmake clean && gmake kanzi'. The Makefile contains all the necessary targets. Builds successfully with C++98, C++11, C++14, C++17, C++20. ### Makefile targets ``` clean: removes objects, libraries and binaries kanzi: builds the kanzi executable kanzi_static: builds a statically linked executable kanzi_dynamic: builds a dynamically linked executable lib: builds static and dynamic libraries test: builds test binaries all: kanzi + kanzi_static + kanzi_dynamic + lib + test install: installs libraries, headers and executable uninstall: removes installed libraries, headers and executable ``` For those who prefer cmake, run the following commands from the top directory: ``` mkdir build cd build cmake .. make ctest ``` By default, the cmake build generates a dynamically linked executable. Choose ```make kanzi_static``` to build a statically linked executable. Credits Matt Mahoney, Yann Collet, Jan Ondrus, Yuta Mori, Ilya Muravyov, Neal Burns, Fabian Giesen, Jarek Duda, Ilya Grebnov Disclaimer Use at your own risk. Always keep a copy of your original files. kanzi-cpp-2.6.0/SECURITY.md000066400000000000000000000016101525576310500152130ustar00rootroot00000000000000# Security Policy Security updates are applied only to the latest release. ## Vulnerability Definition A security vulnerability is a bug that, given a certain input, triggers a crash or an infinite loop. Compression and decompression failures do not belong in this category. ## Reporting a Vulnerability **Please do not report security vulnerabilities through public GitHub issues.** If you have discovered a security vulnerability in this project, report it privately. Please disclose it at [security advisory](https://github.com/flanglet/kanzi-cpp/security/advisories/new). Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: * Operating system * Hardware: CPU, memory * Kanzi version * Command line invoked * Error reported/crash data/log output If possible provide a minimal reproducer. kanzi-cpp-2.6.0/bin/000077500000000000000000000000001525576310500141745ustar00rootroot00000000000000kanzi-cpp-2.6.0/bin/.gitignore000066400000000000000000000001641525576310500161650ustar00rootroot00000000000000**/Debug/** **/Release/** **/*.obj **/*.o **/*.htm **/*.exe **/*.idb **/*.pdb **/*.ncb **/*.sln **/*.suo **/*vcproj*kanzi-cpp-2.6.0/cmake/000077500000000000000000000000001525576310500145045ustar00rootroot00000000000000kanzi-cpp-2.6.0/cmake/KanziConfig.cmake.in000066400000000000000000000002571525576310500203210ustar00rootroot00000000000000@PACKAGE_INIT@ include(CMakeFindDependencyMacro) find_dependency(Threads REQUIRED) include("${CMAKE_CURRENT_LIST_DIR}/KanziTargets.cmake") check_required_components(Kanzi) kanzi-cpp-2.6.0/cmake/cmake_uninstall.cmake.in000066400000000000000000000020641525576310500212660ustar00rootroot00000000000000# Used by smake to uninstall kanzi binary, libraries and header files if(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") message(FATAL_ERROR "Cannot find install manifest") endif() file(READ "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt" files) string(REPLACE "\n" ";" files "${files}") foreach(file ${files}) if(EXISTS "${file}") message(STATUS "Removing file: ${file}") file(REMOVE "${file}") endif() endforeach() set(install_dirs "") foreach(file ${files}) get_filename_component(dir "${file}" DIRECTORY) list(APPEND install_dirs "${dir}") endforeach() # Remove duplicates list(REMOVE_DUPLICATES install_dirs) # Sort deepest directories first list(SORT install_dirs) list(REVERSE install_dirs) foreach(dir ${install_dirs}) if(IS_DIRECTORY "${dir}") file(GLOB children "${dir}/*") list(LENGTH children num_children) if(num_children EQUAL 0) message(STATUS "Removing empty directory: ${dir}") file(REMOVE_RECURSE "${dir}") endif() endif() endforeach() kanzi-cpp-2.6.0/doc/000077500000000000000000000000001525576310500141715ustar00rootroot00000000000000kanzi-cpp-2.6.0/doc/Plot_enwik8.png000066400000000000000000002167201525576310500171120ustar00rootroot00000000000000‰PNG  IHDR° 3ā›9tEXtSoftwareMatplotlib version3.8.3, https://matplotlib.org/ĮæXS pHYsaa¨?§iIDATxœėŨyXTåûđ{Ø÷ME@eßÜ51wK34K4ĶRs)Ķ_ļ§ĸļØbj‹[&ZfŽTæú5wÜÍ TAEA@öķûcbd”eāÅ3÷įē¸œsæ,Ī sOöøž÷($I’@DDDDDDDDÔ@éˆ.€ˆˆˆˆˆˆˆˆčQØĀ""""""""ĸ ,""""""""jĐØĀ""""""""ĸ ,""""""""jĐØĀ""""""""ĸ ,""""""""jĐØĀ""""""""ĸ ,""""""""jĐØĀ""""""""ĸ ,""""""""jĐØĀ""""""""ĸ ,""""""""jĐØĀ""""""""ĸ ,"""™:qâFWWWÁĖĖ AAAøōË/‘‘‘!ēŧ'BDD …č2ęÄíÛˇņÁĀÛÛĻĻĻ°´´„§§'^~ųeœ8qBĩŨŠ+ P(””$ŽØ'DEŸ<ûėŗŨ7++ }ôÜŨŨabb‚f͚á…^ĀéĶ§ëĒ\""ĸ'ŠžčˆˆˆHķ~üņGL˜0xįwāííÂÂBÄÆÆbņâÅ8xđ 6nÜ(ēĖoôčŅčĶ§č24.;;íÛˇGvv6Ūyįāūũû8ū<6l؀„„øûûžyæ7oŪTÛļô˛Ģ 6 uëÖ022ÂĖ™37nÜĀë¯ŋGGGĀÕÕ3gÎDQQ‘Ú1-Z„€€˜™™ÁÜÜžžžøđÃUĪįææbęÔŠĒK,mll‚¨¨¨r5—UŨ÷ëčŅŖčŌĨ LLLĐĸE Ė™3%%%˙ōūĶēuktéŌĨÜúââb4kÖ aaaU~š}û6T:ĒĒlķęáKcbb P(*üqqqQ;Îoŋũ†:ĀÔÔfffčŨģ7âããY[MH’„… "00ÆÆÆ°ļļÆāÁƒqųōeĩíĒō;‘$ M›6ÅĉUûÃÚÚ:::HKKS­˙æ›o §§‡ģwī¨ú%§ .„žžfĖ˜Đ××XZZĒmgee022ĒŪBDD$Cl`ÉHqq1ū÷ŋ˙!88NNNUÚgüøņxīŊ÷ĐŗgODGGcöėŲØēu+:vėˆ[ˇnŠm{ãÆ Œ5 ŖGÆæ͛áįį‡W_}ŗfÍÂ|€wß}ëׯ‡™™ˆëׯ—;ßķĪ?–-[bŨēuˆˆˆĀĻM›ĐģwoĒm‡wŪy“&MÂÖ­[1hРܸqmÛļÅļmÛ0}útüũ÷ßxíĩ×đųįŸcĖ˜1Ē}×ŦYƒ & k׎ظq#6mڄ)SĻ ''GĩÍ˙ũß˙aŅĸEĒã˙ōË/xá…TÍMŊ_Æ ÃđáÍž}ûâƒ>ĀĒUĢyŽQŖFáŸūÁ… ÔÖoßžׯ_Į¨QŖĒü:+ŌĄCĀˆ#°iĶĻĮžæ˛‚‚‚pđāAĩŸŸūúúúđņņQm÷ŲgŸačĐĄđööÆÚĩkņË/ŋāŪŊ{čŌĨ Μ9SåķUÅë¯ŋŽÉ“'ŖGØ´i.\ˆĶ§OŖcĮŽj 'āņŋ…B§Ÿ~;wîTí‹ģwīÂČČģvíR­ßšs'‚ƒƒUĻĮ‘$ S§NÅäɓąlŲ2USÖŲŲĪ=÷æ͛‡Ũģw#;;įΝäI“ĐŧysŧôŌKĩ|‡ˆˆˆd@""""Ų¸qã†@zéĨ—Ē´ũŲŗg%Ō„ ÔÖ>|X }øá‡Ēu]ģv•HąąąĒuˇoߖtuu%ccc)%%Eĩ>!!A }ûíˇĒu3fĖHSĻLQ;×ęÕĢ%ŌĒUĢT뜝%]]]éß˙UÛöõ×_—ĖĖĖ¤+WŽ¨­˙úë¯%ŌéĶ§%I’¤7ŪxC˛˛˛zäk÷õõ•øČmJk.U“÷ëđáÃjÛz{{KŊ{÷~äyoŨē%¨O’$iȐ!RĶĻMĨÂÂBI’Ēö:+3kÖ,ÉĀĀ@ \]]ĨqãÆIĮWÛ.22R %&&Vxœ´´4ŠE‹’tįÎI’$éęÕĢ’žžžôæ›oĒm{īŪ=ÉÎÎN2dHjŽČÁƒ%ŌÜšsÕÖ_ģvM266–Ū}÷]ÕēĒūN–-[&Ž^Ŋ*I’$}ōÉ'’§§§4`ĀiÔ¨Q’$IRAAdjjĒö;zøķ"IĘĪō3Ī<#åææJƒ ’,--Ĩ;w–{Ō˜1cTŋ’ŋŋĨī;‘ļáŦ‡ėŨģũû÷‡ƒƒ 6mÚT­ũK‡Ž?ücjjZ7ÕÂîŨģ#GŽT[ßļm[xyyŠ6”—œĢ–mll`kk‹ĀĀ@888¨Ö{yyŽ\šRîœÃ† S[2dôôôTĩ”ō÷÷‡ģģģÚē?˙üŨēuƒƒƒŠŠŠT?}ûöėŲŗGU˙Ũģw1tčPlŪŧšÜȨŌmūūûoŧ˙ūûˆ‰‰Áũû÷ËŋAŠîûegg‡ļmۖ{]Ŋ/e5jÔũû÷Įʕ+U—ļŨšs›7oƈ# §§Wå×Y™iĶĻáęÕĢXž|9^ũu˜™™aņâÅVģŒōQrrrđĖ3Ī //˙ũˇj$ŌļmÛPTT„#F¨ũžŒŒŒĐĩkWÄÄÄ<ō¸e÷)**‚ôˆ)[˙üķO( >\m;;;”;WU~'=zôÕ(Ŧ;v gĪžčŅŖvėØ@9?UNNŽjÛGš}û6ž~úi9r˙üķēwī^n›ņãĮcũúõ˜7oöėŲƒß~û xúé§ûy!""Ōl`=$''øūûīk´˙ÔŠS‘ššĒöãíí^xAՕ׸qc˜˜˜ 11ąJÛ?j.$‡r—–ŲØؔÛÎĀĀ Üz@^^^šííėėԖõôôШQŖrįĒ¨Ļ´´4üņĮĐ××Wû)Ŋt­´ķōË/cųōå¸rå  [[[´k×NÕ|€oŋũīŊ÷6mڄnŨēÁÆÆ,wŲ^YÕ}ŋ5jTn;CCÃ*5Ë^}õU¤¤¤¨jŽŠŠB~~žZķŦ*¯ķQš6mŠQŖFaņâÅ8qâöėŲŧõÖ[Ũˇ¨¨ƒÆųķįąeËĩKVK/ÛkĶĻMšßÕoŋũöØFÛÃûŦ\š˛ŌmĶŌŌTķV=ŧߥC‡ĘĢ*ŋggg¸ššaįΝČÍÍÅÁƒU ŦäädüûīŋØšs'ŒŅącĮĮžWįĪŸĮáÇŅˇo_øúú–{~ëÖ­ø駟°dÉLž<O=õ† ‚;v ##=‘Üé‰. ĄéÛˇ¯ę_q+RPP€?ūĢW¯ÆŨģwáëë‹/žøĄĄĄ333˜™™Šļ?~ü8Μ9ƒÅ‹×uéDDDĐÕÕE÷îŨņ÷ß#99ŽŽŽÜžôæSSSËm{ũúu4nÜXã5Ū¸q͚5S-áöíÛå M†Ũ¸qcøûûãĶO?­đØeG5 ŖFBNNöîŨ‹3fāŲgŸÅųķįáėė SSSĖœ93gÎDZZšj4V˙ūũqîÜš _ŸīWīŪŊáāā€ČČHôîŨ‘‘‘h׎ŧŊŊÕļ{ÜëŦŽ§žz ŊzõÂĻM›žž[[ÛJˇ;v,víڅ-[ļ @íšŌ÷aŨēuÕŽŽ=ĒļėęęZéļ7†BĄĀž}û*ŧiAEëĒĸ{÷îØŧy3öėŲƒ’’„††ÂÜÜØącvî܉.]ēTéø:tĀ /ŧ€×^{ €râũ˛å'$$P6üʲ˛˛B˖-qęÔŠŊ"""9aĢšF…¤¤$ŦYŗظq#úô郓'OĸUĢVåļ_ļlÜŨŨ+ŧ“Q]øāƒ°eËŒ3›7oV†*UXXˆ­[ˇĸ˙ūxúé§ĢV­RûŸįŖGâėŲŗøčŖ4^ßęÕĢÕ.C\ģv-ŠŠŠT˙ô(Ī>û,ļlŲ777X[[Wé|ĻĻĻčÛˇ/ 0pā@œ>}ē\SĨiĶĻ9r$Ž?Žųķį#77&&&åŽUŸī—ŽŽ.^~ųeĖŸ?ûöíCll,–,YRéöUyĨŌŌŌФIĩ&  ŧ Ā… `bbōȉÉ?ūøcDFFbåʕ^B×ģwočééáŌĨK4hPÕ^p!!!UŪöŲgŸÅœ9s’’‚!C†Tû\•éŅŖ–.]ŠųķįŖ}ûö077 llmܸGÅgŸ}VåãŊōĘ+055Exx8rrr°råJčęęxĐx=tčÚīėöíÛ8ū|…—i6°ĒáŌĨKˆŠŠBrr˛ę/S§NÅÖ­[Yî/1ųųųXŊz5Ū˙}å‘–ęĐĄ-Z„ & 88ãĮ‡ ĨK—Â××ũû÷‡‡‡Ǝ‹īžû:::čÛˇ/’’’0mÚ4899aʔ)¯oÆ ĐĶĶCĪž=qúôiL›6 Uj>Ėš5 ;vė@ĮŽ1iŌ$xxx //IIIØ˛e /^ GGGŒ3ÆÆÆčÔŠėííqãÆ |ūųį°´´T5žÚĩk‡gŸ}ūūū°ļļÆŲŗgņË/ŋ C‡6¯ÔûûõęĢ¯â‹/ž@xx8Œņâ‹/Ē=_•×Y‘_~ųK–,Axx8Ú´iKKK$''cŲ˛e8}ú4ĻOŸ^ŽņYę÷ßĮ§Ÿ~ŠÁƒÃŨŨ‡R=ghhˆÖ­[ÃÅÅŗfÍÂG}„Ë—/ŖOŸ>°ļļFZZŽ9ĸũĻ :uÂØąc1jÔ(ÄÆÆ⊧ž‚ŠŠ)RSSņĪ?˙ĀĪĪãĮ¯öqŸ~úi( lßž]­Ö=zā•W^Q=ŽŽÁƒÃÄăÆũû÷„……aúôé?~<’““„ÔÔT|õÕWČÍÍ­Ō%DDDrĮV5ÄÅÅA’¤rĘæįįW8ŸÂ† pīŪ=Œ1ĸžJ$"" ll´mÛķæÍÃ_|7n@__îîîĮoŧĄÚvŅĸEpssÃO?ũ„~ø–––čĶ§>˙üķ ˙ûV[6l@DD-Z…Būũûcūüų•6LʲˇˇGll,fĪž¯žú ÉÉÉ077‡ĢĢĢĒI]ētÁŠ+°víZÜšs7FįΝņķĪ?ŖI“&” ŠččhĖ›7šššhÖŦFŒņØQTõų~šģģŖcĮŽ8pā† KKKĩįĢō:+ōĖ3ĪāƍØ˛e -Z„;wîĀÜÜūūūøå—_0|øđJ÷=}ú4ååëÖ­S{ÎŲŲIII”#ŊŊŊą`ÁÕü]vvvhĶĻ ƍWÃw¤bK–,AûöíądÉ,\¸%%%ppp@§NĘMØ^U5B`` âããÕUĨKŸ¯Ž~ũúa˖-čßŋ?ž{î9lØ°fff8tč>ũôS,^ŧÉÉÉ°ąąAëÖ­ąhŅ"´oßžF¯ˆˆHNŌŖnëĸå 6n܈~ûí7 6 §OŸV ų.effVnRÚîŨģÃÂÂ7nŦ¯’‰ˆˆŦˆˆĖœ97oŪŦ“šĩˆˆˆˆHž8ĢZˇnââb¤§§?vNĢÄÄDėŪŊŅŅŅõT‘<ąõėėl\ŧxQĩœ˜˜ˆ„„ØØØĀŨŨÆ È#0wî\´nŨˇnŨÂ˙ū÷?øųųĄ_ŋ~Ēũ–/_{{ûGŪŅˆˆˆˆˆˆˆˆ—>$&&Ũēu+ˇū•W^ÁŠ+PXXˆO>ų?˙ü3RRRШQ#tčĐ3g΄ŸŸ ¤¤ÎÎÎ1bDĨˇų&"""""""ĸĒa‹ˆˆˆˆˆˆˆˆ4Ņ= XDDDDDDDDÔ qw(įŦē~ũ:ĖÍÍĄP(D—CDDDDDDD$+’$áŪŊ{ppp€ŽNõĮSąāúõëprr]‘Ŧ]ģv ŽŽŽÕŪ ,æææ”oĸ………ājjŽ¨¨zzü•‰Ä‰Į‰Å ‰ĮRC”••'''UĻēø‰T— ZXX<Ņ ŦÇŖ]ģvĸË ŌjĖ!‘xĖ!‘XĖ ‘xĖ!5d5ē‰“¸QƒÆ–Œ888ˆ.Hë1‡Dâ1‡Db1ƒDâ1‡$Gl`Ɉ‘‘‘čˆ´sH$sH$3H$sHrÄ9°däōåËhŌ¤‰č2ˆ´sH$sH$3HSRR‚‚‚ŅeČÚå˗k>>đôôÄÁƒĢõšVŽ\ …B?˙üŗZûiBMrHDšÅ‰Å RhŌ0q ¨d`â¤ÜŽ9::"&&ˇoßF¯^ŊиqcāģīžÃå˗ŲŗgWë˜÷īßG˙ūũˆącĮâƍĒŸš*áE$'ü¯‹Œ×ŲąãââpčĐ!4oŪüąÛ0 DDD ;;_ũĩęųķįĪ㧟~Âīŋ˙^îîׯ_Į+¯ŧ‚ŋ˙ū^^^ČËËĢÖ4’““ądÉ´oßžĘûiR]搈Ē†9$‹¤:ĄŖ /PŪm ¨Oæū_ŗ&x~Lāū°ŌË ģuë†^x+VŦĀ‡~ˆoŋũAAAøúë¯U˙?Tiii8wîΝ;ĩį$ŠĸIëĪÔÔ´Fû5d%#GŽŠņž ĒQR°°°ĀĖ™3ųųų˜8q".\¨‘Nžģģ;*üך… bøđáĒÉ?ŒŒ`eeUåc;ķæ̓ĄĄa­ë$Ē‰Ú䐈4ƒ9$‹¤:ãtY˜4S_oâ¨\īTûKī*˛bÅ lÚ´ImŊŊ=Ν;‡#GŽ`čĐĄHLLD^^8€ūũûC’$BCC!I’Ú˙×B’$¸¸¸ĀÅÅ’$Uø@õ|éņĒ"''§–¯š¨áaKFĒŌOJJ‚BĄ€‹‹‹ÚúĀĀ@$$$ !!ķæ̓­­-F ˜>}:†WW××ĻP(ĒÔü:sæ îßŋkkk( <˙üķČÍÍÅʕ+áįį===( ŦXąĸÜž‹-‚ÚĩkWã:‰jĢĻ˙JFDšÃ‰Å Rr $ŨwUū9 ąÎšWO*æäˆ—ĘHmnŗZęôéĶ5jūüķO4kÖ ÄŅŖG1gΜZ7**ĒJÛ"&&>>>Øŋ?rrr<õÔS044ÄącĮĘ험˜ˆüû÷ī¯UDĩĨ‰Qí0‡Db1ƒTįttĻĄĸĢhĐĘŪņH.8KFŦ­­U‹‹˜ *JųgqqåûYXX sįÎ8{ö,BCCQPP€6mÚĀÚÚ#FŒĀɓ'áęę ;;;\šrAAAčŲŗ'Ŧ­­áááÃ‡Pm-iUúS:ÔvčĐĄ:tčc_ƒŗŗ3žyæÕå…O?ũ4Ž9‚ &ā‡~€§§'eŖ­ôrĮO?ũÄõë×áåå:t¯Ŋö~üņĮŊ—D5U6‡D$sH$3H$žŽnŨĪFTß8KFΟ?víÚaÃā­ˇ€ääĪ9: AAÖ-\¸ĐŖGüōË/čĶ§ÂÃÃáææ###\šr_|ņÚĩk‡ũû÷#&&ŨēuCaa!‚‚‚Đŧys,_žīŊ÷bbbđÛoŋ!11xįwžž''§Į֝­š¨}Ā€˜={ļęrÃ#GŽ  Ü>>>>øęĢ¯Ôօ‡‡Ģ‡††bęÔŠxöŲgĢüūiBi‰HæH,fHŧŧŧ<˜™™‰.ƒHŖØĀ’™ €Áƒ‡/yNIQŽ˙¯g…””Lœ8đÛoŋ!::įΝí[ˇpãÆ Šö‹‹S;–››žøâ \¸p˗/ĮŋíÚĩCHH^zé%¤§§#""’$!;;[m˙˜˜ >YYY$ ‹-Bnn. k׎čßŋ?"""wīŪEdd¤ĻŪ"""""""zą%#-[zTØŧ”ë `Ö,å˛ąą1tuu!I.]ē„W^y/ŧđ,,, ŖŖƒå˗ÃŅŅũû÷WŽ*åččāÁuÕĨÍ.I’0zôhŦ[ˇoŋũ6fĖ˜ĄÚgîÜšĒĮĄĄĄH.3<ėĖ™3¸~ũ:åķāā`lŲ˛{öėÁôéĶaiiYí÷"&&ĻÚûi‚‡‡‡čˆ´sH$3H$ž‘‘‘čˆ4Ž ,Ųļ5ÉÉV•>/I@jĒōą ĸĸĸĐŗgOôęÕ ;w‡ ŠŠŠ‘‘ ??ŋĘį˙čŖ°bÅ xzz"$$kÖŦĢĢëc‡{{{ÃÛÛû‘ÛÄÅÅ!..—.]ėŨģEEExéĨ—84–”ŒŒ ĩ[$Qũc‰Äb‰Ä+..VÍ+L$œÄ].έ†×wĪã*œP]\…>Æl˜"ģŌ]:tč€ččhÜŋ={öą °páB4nÜß|ķ Ņ¨QŖ*—pāĀe)įΊ&m_´hQ­_DGGcĖ˜18tč 22cƌÁ­[ˇ4r|"MšyķĻčˆ´sH$3H$^aaĄčˆ4N!I]pĻ]˛˛˛`ii‰ĖĖLXXXˆ.§úέú‡tP”ųmCņD(ö F)íŪ „†Ö™DÚāčŅŖhĶĻč2ˆ´sH$3H•ÉËËCbb"\]]y‰[ËÉɁŠŠŠč2H =*įĩíŊp֓Ž¤˜5x¨yē(Ak$` æP΁åäté" N"-Áŋ°‰Į‰Å Š[ąbEŊ_VËæÉXOē›û€mY@%ãčtP‚ąX ÅŨ­ųķ]Ũú+HÛÄÆƊ.Hë1‡Db1ƒ$W׎]Ãk¯ŊĀŲŲoŊõnßž­ÚÆÅÅķįĪWärrrD—@¤ql`=éî§w*ZĀ×áh—‹u뀰°zĢŒH+‹.Hë1‡Db1ƒT׊‹˜ *Jųg}|ä._žŒœ?QQQ¸xņ"/^Œ]ģvĄC‡ČČȨû"ō¨yŽ8SÉXO:c{Āēō§%…æzHŒ=ĘæQ=¨Îˆ¨n0‡Db1ƒT—6l\\€nŨ€đpåŸ..Ęõuiâĉ000ĀöíÛŅĩkW4oŪ}ûöÅΝ;‘’’‚>úĄĄĄ¸rå ĻL™…B…BĄvŒmÛļÁËË fffčĶ§RKo˙ŸČČHxyyÁČČžžžX¸pĄęš¤¤$( Ŧ]ģĄĄĄ022ÂĒUĢpåĘôīßÖÖÖ055…ļlŲÂ;’,ņSũ¤kŌčmŦŠø2B…0zÖ°ãÄWDõÁÖÖVt DZ9$‹¤ē˛a0x0đđāĸ”åúēēâ$##ÛļmçŸ~ cccĩįėėė0lØ0üöÛo¸pá1vėXŒ3FmģÜÜ\|ũõ×øå—_ ŖŖƒáÇcęÔŠXŊz5āĮČ3đũ÷ßŖuëֈĮ˜1c`jjŠW^yEuœ÷Ū{sįÎEdd$ 1vėX`īŪŊ055ř3g`ff}}}ÍŋD‚q֓NG˜žp$ÅCĪ)8Cųŧ'ž"ĒgĪž]‘Öc‰ÄbŠ.oŊUžypĀAˇšđ˛A"˛˛˛D—@¤õ˜C"ą˜AĒ ööšŨŽ:5j„ž={bá…å.ÍģqãV¯^_| …ÕžgĶĻMŅŦY3\ž|-[ļTûquu}ėūNNN7n6l؀ˇß~?ūø#īJ˛Ä–ŒÜē}ûÁo›J$DZZščˆ´sH$3HuĄKĀŅQy“ĒŠ(€““rģēđũ÷ß#??Ŋ{÷ÆŪŊ{qíÚ5lŨē={öDŗfÍđ駟\\\°wī^¤¤¤āÖ­[U>~DD>˙üs,X°įĪŸĮɓ'‰ožųæ‘ûMž<ÛļmCbb"âââđŋ˙ũ^^^(,,ŦÕë%jˆØĀ’“˛ßæl`‘Lčę (?ÜÄ*]ž?_š]]hÕĒbccáææ†_|nnn;v,ēu놃ÂÆÆ0kÖ,$%%ÁÍÍ Mš4ŠōņGe˖aŊđķķC׎]ąbŊĮŽĀ*..Æĉáåå…>}úĀÃà .ŦÕk%j¨’ÄNGVV,--‘™™ ŅåÔ\r˛ōŸ`Đ å}d‰ˆˆˆˆˆ€ŧŧ<$&&ÂÕÕĩÆs4mØ ŧaŲ Ũœ”ÍĢ°0ÍÔID5÷¨œ×ļ÷ÂX2rōÔŠ ėK 'ē"­Į‰Å R] ’’€Ũģ_Uū™˜ČæÕÃrrrD—@¤qzĸ Í),*z°Ā‘œo€H<æH,fęšŽ.*ēŠ†Z‘q–ŒXZY=Xā‘ÖÖÖĸK ŌzĖ!‘XĖ ‘xzzĢBō֌ؕŊg,XDB888ˆ.Hë1‡Db1ƒDâéëë‹.HãØĀ’‘s˙ūû` ,"!NŸ>-ē"­Į‰Å ‰w˙ū}Ņ%iXrRö~˛l`‘L°%#.ŽŽØĀ"ÂÍÍMt DZ9$‹$ĪĐĐPt DĮ–ŒÜĪË{°Ā‘šššĸK ŌzĖ!‘XĖ ‘x%%%ĸK Ō86°d$-=ũÁXDB¤ĻĻŠ.Hë1‡Db1ƒDâŠ.HãØĀ’ļŦˆˆˆˆˆˆ¨!9r$XgĮ˙駟ĐĢW¯:;žœ­XąVVVĒ刈Ē–§NŠI“&Õa˙aKFZˇnũ`#°ˆ„ ]‘Öc‰ÄbInFŽ …B…B}}}´hŅS§NENNNž7&& …wīŪ­öžĻĻĻjËĄĄĄĒ×Pö§¨¨HCՖ—””…B„„ĩõ ,ĀŠ+ęäœųųų˜>}:ĻM›Ļļ>++ }ô<==add;;;ôčŅ6l€ô„˙ŋķÃM§˛ 6mÚ¤ąsŊûîģˆŒŒDbbĸƎYl`ÉČŲsį,<á!$zR:uJt DZ9$‹$9ęĶ§RSSqųōe|ōÉ'X¸p!ĻNZŖcI’T§#¸˙~šucƌAjjĒڏžž^ší ę´6KKËJ.Uņ¨÷oũúõ033C—.]TëîŪŊ‹Ž;âįŸÆ|€¸¸8ėŨģ/žø"Ū}÷]dffÖ¸mckk‹^ŊzaņâÅBÎĪ–Œäåį?X`‹Hˆŧ˛7S "!˜C"ą˜A’#CCCØŲŲÁÉÉ ááá6l˜jdËĒUĢsssØŲŲ!<<éeæ'.IĩmÛ6„„„ĀĐĐûöíƒ$IøōË/ŅĸE # ëÖ­ ŊÔ­[7€ĩĩ5 FŽ @9ĘhŌ¤I°ĩĩ…‘‘:wîŒŖGĒÕ[Ņ$î&&&°ŗŗSû|ōÉ'9r$,--1fĖĘf áâ₹sįĒĪÅÅŸ}ö^}õU˜››ŖyķæXētŠęyWWWĘ+… BCC”ŋ„đQīÃŖŪŋŠŦYŗ P[÷á‡")) ‡Æ+¯ŧooo¸ģģcĖ˜1HHH€™™āΝ;1bŦ­­abb‚ž}ûâ… Ē㔎tÚļmŧŧŧ`ffĻjlĀļmÛ`ddTnÄܤI“ĐĩkWÕōđÔSOÁØØNNN˜4i’j4ßĪ?˙ 333ĩķžųæ›pww¯õˆŋŠFô%$$@ĄP ))ŠĘĮ0`ĸĸĸjUKMą%#––ØĀ"²l‰HæH,f´ąąąjĸô‚‚Ėž=ĮĮĻM›˜˜¨j6•õîģīâķĪ?ĮŲŗgáīī?ū‘‘‘X´hNŸ>)SĻ`øđáØŗgœœœ°~ũzĀŋ˙ū‹ÔÔT,X°@uœõë×cåʕˆ‹‹C˖-Ņģwoddd¨ÎĨĢĢ[­×ķÕW_Á××ĮŽôiĶpėØ1 2/ŊôNž<‰ˆˆL›6­ÜĨsįÎEHHâãã1aÂŒ?įūģ2čȑ#€;w"556l¨đ܏zõūUdßž}j—1—””`͚56lĘmoffĻ…6räHÄÆÆ"::„$IčׯŸÚ„øšššøúë¯ņË/ŋ`īŪŊ¸zõĒj$^=`eeĨúŊ@qq1ÖŽ]‹aÆNž<‰ŪŊ{#,, 'NœĀoŋũ†ūųoŧņ`Ĉčׯ† †ĸĸ"lŨēK–,ÁęÕĢË]*JÛļmqíÚ5\šrĨūO.‘”™™)233E—R+9ŠŠ’¤l]IRĪžĸË!ŌJ999ĸK ŌzĖ!‘XĖ UæūũûŌ™3g¤û÷ī?X,I͚Õ˙Opp•ë~å•W¤įž{Nĩ|øđaŠQŖFŌ!C*Üūȑ#éŪŊ{’$IŌîŨģ%ŌĻM›TÛdggKFFFŌÔö}íĩפĄC‡ĒíwįÎĩũôõõĨÕĢWĢÖHŌ—_~ŠZWTT¤vÜŽ]ģJúúú’ŠŠŠęį˙ūī˙$I’$gggiāĀjۇ‡‡K=úĘwŪyGōööV-;;;KÇW-—””HļļļŌĸE‹$I’¤ÄÄD €¯vœ˛īguŪ‡˛ī_EîÜš#öîŨĢZ—––&žųæ›Gî{ūüy €´˙~Õē[ˇnIÆÆÆŌÚĩk%I’¤ČČH €tņâEÕ6?üđƒÔ´iSÕō¤I“¤§Ÿ~ZĩŧmÛ6ÉĀĀ@ĘČȐ$I’^~ųeiėØąjįŪˇoŸ¤ŖŖŖĘEFF†äčč(?^jÚ´ŠôÉ'Ÿ<˛öŌēĘūnKH7n”$ŠâĪS||ŧ@JLLTËŌŌRõüŒ3¤€€ĩķ•öObbb*Ŧ§Âœ?´oM{/å/xĨ'Öé3gĐĻt#°ˆ„8yō$Úĩk'ē "­Æ‰Å Rĩܸ¤¤ˆŽâąūüķO˜™™Ą¨¨………xîšįđŨwßâãい„ddd¨.ßģzõ*ŧŊŊUĮ(;2čĖ™3ČËËCĪž=ÕÎSPP ~sŽ‡\ēt ………čÔŠ“jžž>Úļm‹ŗgĪĒÖŨŋ_ui\ŠaÆáŖ>R-—‡ęá›/œ={Ī=÷œÚēN:aūüų(..Vđ*;JĄPĀÎÎNíōÉĮŠÎûđ¸D”Îûedd¤Z'ũ÷˙Å …â‘ûž={zzzjß]5‚‡‡‡Úûjbb777Õ˛ŊŊŊÚë6l:tč€ëׯÃÁÁĢW¯Fŋ~ũ`mm 8vė.^ŧˆÕĢWĢÕXRR‚ÄÄDxyyÁÚÚ?ũôz÷;âũ÷ßdí`nnŽ¸¸¸rë[ĩjõØ}ĢËØØ€r4Z}cKNʆ’ ,"""""jčū›‡ŠĄŸˇ[ˇnX´hôõõáāā}}}@NNzõę…^ŊzaÕĒUhŌ¤ Ž^ŊŠŪŊ{—› Ŋė%`ĨMŽŋūú ͚5SÛÎĐĐ°Ō:*kČH’ôØ&ĨĨ%ZļlYás_žVŅņ¤ ūŗô}(ĨP(*œĢ2Õyw ]ŖF P(pįÎÕē&MšĀÚÚZ­ U‘Š^[éú˛īCE¯ˇėžmÛļ…››ÖŦYƒņãĮcãƍˆŒŒT=_RR‚×_“&M*wŽæ͛ĢīŨģēēē¸~ũ:rrr`aaņČúutt*ũŨ–ŨĻô5•*{ydU•^ĒÚ¤I“jī[[l`ÉHsgį l` áââ"ē"­Į‰Å RĩÄƊŽ JLMM+lœ;wˇnŨœ9sāä䈭Âkōöö†ĄĄ!Ž^ŊĒ6ÁwY”ķ(•jŲ˛% đĪ?˙ <<€˛ ‹É“'Ģļ{TŦ*ŧŊŊņĪ?˙¨­;pāÜŨŨĢ<ŋVEõWtžĮŊUe``oooœ9sŊzõ lÚŧøâ‹øå—_0cƌrķ`åääĀĐĐŪŪŪ(**ÂáÇŅącGĀíÛˇqūüyxyyUĢŽđđpŦ^ŊŽŽŽĐŅŅÁ3Ī<Ŗz.((§OŸ~dŗéĀøōË/ņĮāũ÷ßĮ›ož‰•+WVĢ†Š”6œRSSU#ÂĒ}œS§NA__>>>ĩŽŠē8‰ģŒ—ítŗE$D]ß™ˆ9$‹$mŌŧysāģīžÃå˗Ųŗg?v?sssL:SĻLÁʕ+qéŌ%ÄĮĮã‡~P5+œĄP(đįŸâæ͛ČÎΆŠŠ)ƏwŪy[ˇnř3g0fĖäææâĩ×^Sŋ˛EUõöÛoc׎]˜={6Ο?•+WâûīŋWMX^ļļļ066ÆÖ­[‘––†ĖĖĖŊÕŅģwīrˇĪ>û NNNh׎~ūųgœ9s.\Āōåˈėėl´jÕ Ī=÷ƌƒūųĮĮđáÃŅŦYŗr—R>ΰaÇO?ũƒVģ¤ņŊ÷ŪÃÁƒ1qâD$$$ā… ˆŽŽÆ›ož ¸wī^~ųeŧųæ›čÛˇ/~ũõWŦ]ģŋ˙ū{ĩߋ‡ĩlŲNNNˆˆˆĀųķįņ×_•ģŗdUėÛˇ]ētQ]JXŸØĀ’‘ä˛×ŽŗE$Drr˛čˆ´sH$3HÚ¤I“&Xąb~˙ũwx{{cΜ9øúë¯Ģ´īėŲŗ1}út|ūųįđōōBīŪŊņĮĀÕÕĐŦY3Ėœ9īŋ˙>š6mĒēSŨœ9s0hĐ ŧüōË ÂŋąmÛ6Õ¨å._ŦŽ   Ŧ]ģkÖŦ¯¯/ĻOŸŽYŗfUxwÅĘčééáÛoŋŒ%KāāāPi#čqīCuŒ3[ļlQk–Y[[ãĐĄC>|8>ųä´nŨ]ētATTžúę+՝S###ŒgŸ}:t€$IØ˛eKšË§UĢVhĶĻ Nœ8Ąēû`)ėŲŗ.\@—.]ĐēukL›6 ööö€ˇŪz ĻĻĻøėŗĪ>>>øâ‹/0nÜ8¤ÔrŽ8}}}DEEáÜšsĀ_|O>ų¤ÚĮ‰ŠŠÂ˜1cjUKM)¤Úļfe ++ –––ČĖĖ|ėĩĨ Ų‘Ŋ{ŅļtØe׎@LŒĐzˆ´ŅáÇ9q-‘`Ė!‘XĖ U&//‰‰‰puuU•Bš—]nwm1dČ´nŨ|đčRdį¯ŋūÂ;īŧƒ'N@O¯âŠ•ķÚö^8KFZ=X`_’Hˆ ˛9$"!˜C"ą˜A"ņLLLD— ĖW_}ĨĩÍģē–““ƒČČČJ›Wu ,ų÷üų l` qîÜ9Ņ%i=æH,fHŧŧŧ<Ņ%ãėėŦšSŠ4kȐ!BGØ˛%#š÷ī?X`‹HˆÜÜ\Ņ%i=æH,fHŧ’˛7ø"’ 6°dÄ´ė0I6°ˆ„āpe"ņ˜C"ą˜A"ņtuuE—@¤ql`Ɉ[˖ØĀ"ÂÍÍMt DZ9$‹¤Įá}ÄꞡĄĄčHKÕežŲĀ’‘ã'Nˆ.Hë?~\t DZ9$‹¤Ę”Ž *((\‰üņR^Ĩôŗ§¯¯¯ņc‹™:žę˙Uƒˆˆˆˆˆ===˜˜˜āæ͛Đ×ׇŽĮSԕüü|awŠ#í$Irss‘žž++Ģ:šŒ•Ÿhiîėü` ,"!š7o.ē"­Į‰Å Re ėí푘˜ˆ+WŽˆ.G֊‹‹9 aee;;ģ:96XrĸPXā,"!ÔrHDB0‡Db1ƒDâ1‡$Gl`Ɉ‘‘/!$ĖČČHt DZ9$‹$9$9RH;YYY°´´Dff&,,,D—ScÅÅÅĐmܸ{pwūũWtIDZ§¸¸ēēēĸË ŌjĖ!‘XĖ ‘xĖ!5DĩíŊp–ŒÄÆÆr‘`ąąąĸK ŌzĖ!‘XĖ ‘xĖ!ÉXrÃÉ X2booĪ‘`öööĸK ŌzĖ!‘XĖ ‘xĖ!ÉX2bbb‘`&&&ĸK ŌzĖ!‘XĖ ‘xĖ!ÉX2réŌ%6°ˆģté’čˆ´sH$3H$sHrÄ–Ü°EDDDDDDD2ÖŒøøø°E$˜čˆ´sH$3H$sHrÄ–Œ\ŋ~ ,"ÁŽ_ŋ.ē"­Į‰Å ‰Į’ą%#wîÜa‹H°;wîˆ.Hë1‡Db1ƒDâ1‡$Gl`Ɉžž>XD‚éëë‹.Hë1‡Db1ƒDâ1‡$G Ib§#++ –––ČĖĖ„………črjĮŅHIš5’“EWCDDDDDDDTëŪ G`ÉČáÇ9‹H°Ã‡‹.Hë1‡Db1ƒDâ1‡$Gl`É XDDDDDDD$3l`ÉHĶĻM4°ˆHˆĻM›Š.Hë1‡Db1ƒDâ1‡$Gl`ɈÚ5¤E$Ä?‘ 0‡Db1ƒDâ1‡$Gl`Éȅ x !‘`.\]‘Öc‰Äb‰ÄcIŽØĀ’6°ˆˆˆˆˆˆˆHfØĀ’///6°ˆķōō]‘Öc‰Äb‰ÄcIŽØĀ’‘ôôt6°ˆKOO]‘Öc‰Äb‰ÄcIŽtëķĪ?‡BĄĀäɓšŨž={ ###´hŅ‹/ŽŸ˜ÛˇoŗE$ØíÛˇE—@¤õ˜C"ą˜A"ņ˜C’ŖÛĀ:zô(–.] ˙Gn—˜˜ˆ~ũúĄK—.ˆĮ‡~ˆI“&aũúõõTiÃĄĢĢË‘`ēēēĸK ŌzĖ!‘XĖ ‘xĖ!ÉQƒl`eggcØ°aøņĮammũČm/^Œæ͛cūüųđōōÂčŅŖņęĢ¯âë¯ŋŽ§jŽ6°ˆ ]‘Öc‰Äb‰ÄcIŽdkâĉxæ™gĐŖGĮn{đāAôęÕKm]īŪŊ‹Â ÷ÉĪĪGVV–ڏ=z” ,"ÁŽ=*ē"­Į‰Å ‰Į’é‰.āakÖŦA\\\•wãÆ 4mÚTm]ĶĻMQTT„[ˇnÁŪŪžÜ>Ÿū9fΜYn}ll,LMM„ŗgĪâūũû077‡ĢĢ+Nœ8pvvFII Ž]ģ Äŋ‘ SSS¸ģģ#>>āčč]]]\šrāīī¤¤$deeÁČČ>>>8vėĀÁÁFFF¸|ų2Ā××ÉÉɸ{÷. ˆ#GŽėėė`ff†‹/PŪa"-- ˇoßF^~>ŒáØáÃhŌ¤ Ŧ­­qūüy€‡‡222pķæMččč M›6ˆEqq15j[[[œ={ĐĒU+dee!-- ĐŽ];ÄÅÅĄ°°ÖÖÖpppĀéĶ§nnnČÍÍEjj*eĮ˙ÔŠSČË˃ĨĨ%š7oŽ“'O\\\PTT„ääd@PPΝ;‡ÜÜ\˜™™ÁÍÍ Į4oŪpõęU@@@.]ē„ėėl˜˜˜ĀĶĶqqqĒ÷[OOIII???\Ŋz™™™022‚¯¯/bccööö011ÁĨK—>>>¸~ũ:îÜš}}}áđáÃĒĪ“…….\¸ zŋĶĶĶqûömčęę"$$GEII š4iüûīŋwwwÜšs7oŪ„BĄ@ÛļmqėØ1ÁÆÆM›6UŊß-[ļDvv6nܸhÛļ-PPP+++8::âÔŠS€-Z //ׯ_ãôéĶČË˃……\\\Ô>ŗÅÅÅĒ÷ģuëÖ8ūŗĨīwãƍŅ¸qcœ;wNõ™ÍĖĖTMZYö3kcc;;;œ9sFõ™ÍÉÉQŊßmÚ´Á‰'ŸŸ+++899Š>ŗŽŽŽ(((@JJŠę3[ŲwDéīĨ!~Gddd@OOÁÁÁ8rä$Iâwŋ#dų‘‘‘Ã‡7Čīˆ†ü÷~Gđ;BSߡoßÆáÇėwDCū{ŋ#øĄŠīˆœœdee5Čīˆ†ū÷~GÔŨwDEũ™ęPHRÃĒsíÚ5„„„`ûö턆†"00ķįĪ¯pwwwŒ5 |đjŨūũûŅšsg¤ĻĻÂÎÎŽÜ>ųųųČĪĪW-geeÁÉÉ ™™™°°°Đė‹ĒG—/_F‹gžΝ,,€ĖLŅ%iË—/ŖE‹ĸË ŌjĖ!‘XĖ ‘xĖ!5DYYY°´´ŦqīĨAĀ:vėŌĶĶŦZW\\ŒŊ{÷âûīŋG~~~šÉčėėėT]ÍRéééĐĶĶCŖF*<ĄĄ! 5˙ŗąąá%„D‚ŲØ؈.Hë1‡Db1ƒDâ1‡$G jŦîŨģãäɓHHHPũ„„„`Ø°aHHH¨đN :tĀŽ;ÔÖmßž!!!Đ×ׯ¯Ō„˙ũ— ,"ÁJ‡t‘8Ė!‘XĖ ‘xĖ!ÉQƒenn___ĩuĻĻĻhÔ¨‘jũ|€””üüķĪ€qãÆáûīŋĮ˙ũß˙aĖ˜18xđ ~úé'DEEÕ{ũ XDDDDDDD$3 jVU¤ĻĻĒ&Ų”žmŲ˛111 ÄėŲŗņíˇßbĐ AĢÃŨŨ ,"ÁÜŨŨE—@¤õ˜C"ą˜A"ņ˜C’Ŗ5Ģ"111jË+VŦ(ˇM׎]Uw+Đfwî܁5XDBŨšsÖÖÖĸË ŌjĖ!‘XĖ ‘xĖ!ÉŅ7‹*wķæMŽĀ"ėæ͛ĸK ŌzĖ!‘XĖ ‘xĖ!ÉX2ĸ(m^‘0Ė!‘xĖ!‘XĖ ‘xĖ!ɑB’8T'++ –––ČĖĖ„………črj'08~04ōōDWCDDDDDDDTëŪ G`ÉČącĮx !‘`ĮŽ]‘Öc‰Äb‰ÄcIŽØĀ’‘ĸĸ"6°ˆ+**]‘Öc‰Äb‰ÄcIŽØĀ’6°ˆŗąą]‘Öc‰Äb‰ÄcIŽØĀ’‘ĻM›˛E$XĶĻME—@¤õ˜C"ą˜A"ņ˜C’#6°däėŲŗl` vöėYŅ%i=æH,fH<æäˆ ,ša‹ˆˆˆˆˆˆˆd† ,iŲ˛%XD‚ĩlŲRt DZ9$‹$9$9bKF˛ŗŗ4°ˆHˆėėlŅ%i=æH,fH<æäˆ ,šqã†z‹Ŗ°ˆęŨ7D—@¤õ˜C"ą˜A"ņ˜C’#6°ä† ,"""""""’…$ąË‘••KKKdffÂÂÂBt95&I;(Wēēb‹"Ō2’$AÁKy‰„b‰Äb‰ÄcŠ!Ēmī…#°d$!!#°ˆKHH]‘Öc‰Äb‰ÄcIŽØĀ’‘‚‚6°ˆ+((]‘Öc‰Äb‰ÄcIŽØĀ’+++6°ˆŗ˛˛]‘Öc‰Äb‰ÄcIŽØĀ’GGG6°ˆstt]‘Öc‰Äb‰ÄcIŽØĀ’‘S§NŠ7°ˆ¨Ū:uJt DZ9$‹$9$9bKÎ8‹ˆˆˆˆˆˆˆd€ ,iŅĸ/!$ŦE‹ĸK ŌzĖ!‘XĖ ‘xĖ!ÉX2’——Į‘`yyyĸK ŌzĖ!‘XĖ ‘xĖ!ÉX2rũúu6°ˆģ~ũēčˆ´sH$3H$sHrÄ–Ü°EDDDDDDD2ÖŒŗE$Xpp°čˆ´sH$3H$sHrÄ–Œœ>}š ,"ÁNŸ>-ē"­Į‰Å ‰Į’ą%#œÄH|0räHÕHĢŌŸ‘#G†ŠĄC‡Öøe%&&âĶO?ÅčŅŖ1{öl@BBBGDDDDDDDO6°d¤Yŗfę ŦĮX¸p!ƏŪŊ{cûöí°˛˛‚Ūzë-,X°&LĀîŨģ1zôhĩũbccakk‹°°0œ?īŊ÷`üøņˆŠŠÂʕ+akk čØąãcëČÎÎÆ­[ˇpëÖ-dffV¸ĄĄĄęqtt4 GU~­DõĨYŗfĸK ŌzĖ!‘XĖ ‘xĖ!ɑžčHs ÔWQ}JLLT@$"1˜C"ą˜A"ņ˜C’#^B('R1`}čĀ @IQĨ›ÚØØ`˖-(**B¯^ŊI’ P(PTT„ŒŒ Ŧ]ģųųų€ĩk×>ōÔ …īŧķVŦXOOO„„„`͚5ĒųąJÂÆÆŽŽŽ D`` ŌŌŌpõęUL™2íÚĩÃ÷ß_îø­[ˇÆÚĩkadd„­[ˇbá…øã?YS^^wwwĸOŸ>HJJzä>DDDDDDDÔđ°%įVŖ͂>ĀŠŋCŌĖîœ[]é.:t@tt4îßŋž={â… X¸p!7nŒožųŽŽŽhÔ¨€Į7°¨šUįΝSMÚžhŅ"ĩm0`Āŧ˙ūûHHH@BBēuë†āā`Ŧ]ģááá厯šH>//)))˜4iŪ|ķÍĮÖ4vėXüûīŋHHHĀŗĪ>[ĨK‰jÃĪĪOt DZ9$‹$9$9bKέú‡âˇ{@ Ę?ĪúWkbš¸¸ā¯ŋū‚••ņ˙÷$ ß|ķ š5k†ß˙M›6EŖF””„3gÎĀÉÉ ;wîÄäɓņúë¯#&&6l@Ÿ>}Đž}{Lœ8đ÷ßC’$ĩŸ+V€jš2đōō‚ŽŽō#I’ŠšsįâĮ,wėĮĻ222Bŋ~ũ øī˛ĘöíÛãōåË5}—‰ĒäÚĩkĸK ŌzĖ!‘XĖ ‘xĖ!ÉXOē’b`Öā  x¸?$¸åķ%ÅĒÕũúõS~š:u*ŧŧŧ†UĢVÁÅÅ'OžÄ‰'0wî\ØÚÚbÖŦYčŅŖ°xņb¤§§cĖ˜1Øŧy3<¨v‡ĀēpæĖ\Ŋz]ģvEëÖ­1mÚ4?~Į‡|ûíˇčßŋTHôĀŨģwE—@¤õ˜C"ą˜A"ņ˜C’#6°žt7÷Û˛”ÍĒŠHP>s_š§vīŪˆˆüõ×_077GûöíąuëVŧũöۈŽŽ†ŠŠi…‡ũôĶ:Ē’HŠŽēDôxĖ!‘XĖ ‘xĖ!ÉXOēûŠĀĮlsįŋíĘ8uęF…Í›7ÃÁÁ€rNŦ„„´k×ëׯG›6m*éô¨KÖącGĸ]ģvUŪįaÎÎÎ4hŒabb‚°°09r¤Üv?˙üŗjbøČČHÕú¯ŋū6lĀß˙ “×ATūūūĸK ŌzĖ!‘XĖ ‘xĖ!ÉXO:c{Āēō§%@ųŧąŊj]JJ ˆČČHøøø¨Ö'&&ÂĖĖ C† Áwß}‡ķįĪ#;;ČĖĖTmץCÄĮĮãüųķ€e˖Uzū !!ĄÜŨĢ#<<ÛˇoGII Š‹‹ącĮ”ÛnĈĒK#Gøæ›o…;vĀĘĘĒÆ5UÕŅŖGE—@¤õ˜C"ą˜A"ņ˜C’#6°žtMēŊ-EÅO—@gŧƒ•ÛũgŲ˛e¸yķ&ĻL™ĸą˜˜#00:uÂW_}KKKtīŪ999Ā¸qã`kk‹ĨK—ĸ˙ūčØąŖjâõÚXĩjņûīŋcÚ´ipttD||<āĨ—^‚ƒƒ|||{{{ŧņƏ=frr2Ū~ûmÜŊ{ŨēuĢõH0"""""""C!Uįz0™ĘĘʂĨĨ%233aaa!ēœę;ˇwēž‹ô,č  (G^•@ D(b°rŊ9ÂÂDJ$WŽ\ŗŗŗč2ˆ´sH$3H$sH Qm{/%Å­†ĄŊâ_D`’áˆbč Žˆ@ēbræ˜<¨Áûˆ¨š*ģųÕæH,fH<æäˆ ,Øˇ8Ÿf‹O0Íq z(Fs\Ã'˜†˜A’€kהÛQŨēté’čˆ´sH$3H$sHrÄ– ¤Ļ>~›ęlGDDDDDDDԐ°%öößĻ:ÛU•BĄ@vvv­3ū|¤§§Ģ–#""0uęT@tt4Ūyį*įÃ?„——Đļm[üī˙ĢumDÕåíí-ē"­Į‰Å ‰Į’ą%]ēŽŽ€BQņ|ü āä¤ÜN„ĸĸĸG>˙pĢŦāĢ¯žĒŌyēt邸¸8?~?ūø# „ŧŧŧj×KT7nÜ]‘Öc‰Äb‰ÄcIŽØĀ’]]`Á@’ԛX …ōĪųķ•ÛiÚ×_N:ÁŨŨQQQeÎĢĀÜšsŠ>øiiixūųįáįį___,]ē0kÖ,\ŋ~ƒF`` ԎŋbÅ <¸JĩôíÛÆÆÆ???ãÖ­[šyĄDU”‘‘!ē"­Į‰Å ‰Į’é‰.€4#, øęĢD,˜iälkÕzGGeķ*,ŦnÎĢP(°˙~\ž|mÛļEįΝáääČĪĪGLL āÅ_„§§'6n܈ôôt#00Ķ§OĮōåËąnŨ:øúú6mÚTëē"##áææGGĮZ‹¨:ôõõE—@¤õ˜C"ą˜A"ņ˜C’#ŽĀ’‘ŠS[ éĩO°ĄøCąûģSHLŦģæŒ=ĐĸE tîÜûĘÜęđÕW_U=Ūšs'&Nœ°ĩĩEXXvíÚU'5íÚĩ 3gÎĚ5kęäøD$ē"­Į‰Å ‰Į’ą%#‡†Ž.Š=Š5mYŖË‹‹˜ *JųgqqÕ÷U”^ŗĀĖĖŦŌį*ZŽŽ3gÎ 00ĒÆėŲŗŖFÂüŸ¨Ļ>,ē"­Į‰Å ‰Į’ą%7e›BRœē?ʆ €‹ Đ­ŽüĶÅEšž"˗/$%%áŸūAįΝ+ÜŽGĒy¯nŪŧ‰7â駟XXX 33ŗZuz{{#!! øá‡{÷îÅË/ŋŒÍ›7#  ZĮ#""""""ĸ†‹s`Ɉ­­m­X6ƒ—ß-%Eš~Ũēō—#ĸS§N¸yķ&žûî;ÕüWûöÛo1nÜ8øûûŖ¤¤}ôÚļm ˜4iFŦXąĸZ5—õÚk¯!??ŖFR­ûå—_āįįWãcU—­­­čˆ´sH$3H$sHr¤¤ Ķ‘™ŦŦ,XZZ"33ĸËŠąŒŒ ØĖ™|õ•rŞ=ĀSOUißâbåHĢä䊟W(”Â'&ÖÍŨ ‰ä"##666ĸË ŌjĖ!‘XĖ ‘xĖ!5DĩíŊđBšpáBG`íÛWyķĒôP׎)ˇ#ĸĘ]¸pAt DZ9$‹$9$9bKnjØĀJMÕėvDDDDDDDDšÂ–ŒxzzÖ¸eo¯Ų툴•§§§čˆ´sH$3H$sHrÄ–ŒÜēuĢÆ Ŧ.]”s\•ŨŊ,…prRnGD•ģuë–čˆ´sH$3H$sHrÄ–ŒÔĻĨĢ ,X |üpĢtyū|NāNô8üË‘xĖ!‘XĖ ‘xĖ!ÉX2ĸŖŖSã„…ëÖ͚Š¯wtTŽ Ķ@‘TŠëׯŖ[ˇnĩ>NNNF???xxxāũ÷ßo6ZttøĩJ$sH$3H$sHrÄOĩŒ´iĶĻV ,@Ų¤JJvī~ũUųgb"›WõÁÁÁģwīŽõq>ûė3Ā‰'pęÔ)ÄĮĮcŨēuĩ>.UM›6mD—@¤õ˜C"ą˜A"ņ˜C’#6°däčŅŖĩn`ĘËCCĄC•ō˛ÁšYŋ~=<==Ņēuk|ōÉ'P(ČÎÎƖ-[¨ú122Âʕ+‘””„ƍĢöW(ˆˆˆ@§Nāî¨¨*÷øņãčÛˇ/ ôõõŅĢW/üōË/uõ2é!G]‘Öc‰Äb‰ÄcIŽØĀ’‘’’4°¨öŌĶĶ1vėXüņĮˆ‡™™™ęš~ũú!!! ˜:u*ŧŧŧVÉ7…Bũû÷cëÖ­xķÍ7qíÚĩĮžģM›6Xģv- pīŪ=lܸIIIšziô%%%ĸK ŌzĖ!‘XĖ ‘xĖ!ÉX2Ō¸qc6°ˆC‡!((­ZĩŒ5ĒÜ6ģwīFDDūúë/˜››WxœŅŖGZ´hÎ;cßž}=÷{īŊ'''´mÛ @ĮŽĄ¯¯_‹WCÕQv‰Á‰Å ‰Į’é‰.€4‡ ŦúS\ ėÛ¤Ļöö@—.ę—ZJ’ÅÃˇs,ãÔŠS5jūúë/888TųŧŗcĮŽČÍͅĄĄ!> ###Ė›7Oõüœ9sāíí]åsPíđ/ Dâ1‡Db1ƒDâ1‡$G%#įΝ]‚VØ°pqēuÂÕē¸(חjßž=Ž;†‹/VŽ\Šz.%%Ddd$|||yŽå˗’’’đĪ?˙ sįÎåļ9pāpøđa@VVrss‰‰‰X´hŪ~ûíZŧbĒæH<æH,fH<æäˆ#°ä†#°ęԆ ĀāÁåßڔåúuë”wllÚ´)/^Œgžy5B˙ūũĄ¯¯Ė;7oŪĔ)STûĪš5 ūūūåÎghhˆN:áæ͛øîģīāääôØ/_žŒ!C†@OOzzz˜7okûŌ‰ˆˆˆˆˆˆ„QHģYYY°´´Dff&,,,D—Sc°ųá`útåŠ?˙žyFlQ2R\Ŧi•œ\ņķ āč$&*/'ŧwīžjnĢČČHüôĶOøįŸĒ|>…B{÷îŠMO _FFlllD—A¤Õ˜C"ą˜A"ņ˜CjˆjÛ{á%„2’™™ÉXuh߾ʛW€ōížvMš|ûíˇ „¯¯/"##ņã?ÖOĄ$TffĻčˆ´sH$3H$sHrÄ–Œ¤§§ŗU‡RSĢˇŨG}„„„œ:u {÷—WĩÎ'IG_=ŌĶĶE—@¤õ˜C"ą˜A"ņ˜C’#6°ä† Ŧ:co¯Ų툈ˆˆˆˆˆ¨jØĀ’‘víÚąU‡ētQÎqUö-.KĄœœ”Û‘öj׎čˆ´sH$3H$sHrÄ–ŒÄÅÅąU‡tu ”nb•.ĪŸ¯ÜŽ´W\\œčˆ´sH$3H$sHrÄ–Œ˛UĮ€uë€fÍÔ×;:*ׇ…‰Š‹ŽÂÂBŅ%i=æH,fH<æäHOt¤9666l`Ճ°0āšį”wLMUÎyÕĨ G^‘oWL$sH$3H$sHrÄ–ŒØŲŲąUOtuĐPŅUPCdgg'ē"­Į‰Å ‰Į’ņB9sæ XD‚9sFt DZ9$‹$9$9bKnØĀ""""""""™aKFÜÜÜØĀ"ĖÍÍMt DZ9$‹$9$9bKFrrrØĀ",''Gt DZ9$‹$9$9bKFnܸÁ‘`7nÜ]‘Öc‰Äb‰ÄcIŽØĀ’6°ˆˆˆˆˆˆˆHfØĀ’‘6mÚ¨7°ˆ¨ŪĩiĶFt DZ9$‹$9$9bKFNœ8Ąž‚#°ˆę]šQŊc‰Äb‰ÄcIŽØĀ’‘üü|^BH$X~~žčˆ´sH$3H$sHrÄ–ŒXYYąE$˜•••čˆ´sH$3H$sHrÄ–Œ899ąE$˜“““čˆ´sH$3H$sHrÄ–Œœ>^t DZ9$‹$9$9jp—€‡‡p÷î]Ŧ_ŋ¯ŧō öėŲSaËÃÃĒå:āÚĩkøúë¯ņÔSOUxüĪ?˙3gÎ,ˇ>66ĻĻĻ ÂŲŗgq˙ū}˜››ÃÕÕ'Nœ8;;Ŗ¤¤׎] ŧģÃŋ‘ SSS¸ģģĢž,ĄĢĢ‹+WŽüũũ‘””„ŦŦ,ÁĮĮĮŽ888ĀČČ—/_øúú"99wīŪ…qäČ€ĖĖĖTķyyy!-- wîÜÁĨ˗áößëšrå Š/_†ĩĩ5Ο?¯zŋ222pķæMččč M›6ˆEqq15j[[[œ={ĐĒU+dee!-- ĐŽ];ÄÅÅĄ°°ÖÖÖpppĀéĶ§nnnČÍÍEjj* $$§NB^^,--Ņŧysœ>>¸~ũ:îÜš}}}áđáÀĻM›ÂÂÂ.\PŊßééé¸}û6tuu‚ŖGĸ¤¤Mš4 ūũ÷_Ęų¸sįnŪŧ …BļmÛâØąc(**‚ š6mĒzŋ[ļl‰ėėlܸqĐļm[$$$   VVVpttÄŠS§-Z´@^^Ž_ŋÆéĶ§‘—— ¸¸¸¨}f‹‹‹UīwëÖ­qūüyäääĀĖĖ -[ļDBBĀÉÉ :::jŸŲÄÄDÜģwÆÆÆđōōRŊß͚5ƒUī÷ĩk×p÷î]ÂßßGU}fMMMUīˇˇˇ7nܸŒŒŒrīˇ­­-,--Uīˇ§§'nŨē…[ˇnŠ>ŗĨīwãƍŅ¸qcÕMZĩj…ĖĖL¤§§—ûĖÚØØĀÎÎgΜQ}fsrrTīw›6mpâÄ äįįÃĘĘ NNNĒĪŦĢĢ+ ’’ĸúĖVöQú{iˆßĐĶĶCpp0Ž9I’ФI~Gđ;BvßwīŪÅáÇäwDCū{ŋ#øĄŠīˆ;wîāđáà ö;ĸ!˙=‚ßüŽĐÔwDNN˛˛˛äwDC˙{ŋ#ęî;ÂŪŪĩĄ¤†?L§GpssÒ%KĒ´ũ§Ÿ~ŠUĢVŠÂú°üü|äįįĢ–ŗ˛˛āää„ĖĖLĩ‰āŸ4)))hvč0x°rŗ_īŧ#ļ("-“’’‚f͚‰.ƒHĢ1‡Db1ƒDâ1‡ÔeeeÁŌŌ˛ÆŊ—9ëa’$Š5œ'>>ū‘=CCCjĸ´EWW—— ĻĢĢ+ē"­Į‰Å ‰Į’5¸և~ˆž}ûÂÉÉ ÷îŨÚ5kƒ­[ˇ>øā¤¤¤āįŸĖŸ?...đņņAAAV­Z…õë×cũúõ"_†WŽ\XDB]šrvvvĸË ŌjĖ!‘XĖ ‘xĖ!ÉQƒk`ĨĨĨáå—_Fjj*,--áīī­[ˇĸgĪž€ÔÔTÕ5ĘPPP€ŠS§"%%ÆÆÆđņņÁ_ũ…~ũú‰z bąEDDDDDDD2ķDĖU×j{fCq˙ū}oß ¨\ņŲgĀ­‰HÛÜŋÆÆÆĸË ŌjĖ!‘XĖ ‘xĖ!5DĩíŊčÔAM$HRRG` Vzg"‡9$‹$9$9bKF˛˛˛ÔXDTī˛˛˛D—@¤õ˜C"ą˜A"ņ˜C’#6°dÄČČH}G`Õģr9$ĸzĮ‰Å ‰Į’ą%#>>>ŧ„H0Ņ%i=æH,fH<æäˆ ,9vėXD‚;vLt DZ9$‹$9$9bKnØĀ""""""""™ŅĶÄAōķķqäČ$%%!77Mš4AëÖ­áęęĒ‰ÃS988V°ETīD—@¤õ˜C"ą˜A"ņ˜C’ŖZ5°8€īžû›6mBAAŦŦŦ`llŒŒŒ äįįŖE‹;v,ƍsssMÕL•022â,"Á8a&‘xĖ!‘XĖ ‘xĖ!ÉQ/!|îšį0xđ`4kÖ ÛļmÃŊ{÷pûöm$''#77.\ĀĮŒ]ģvÁŨŨ;vėĐdŨTË—/ŗE$Øå˗E—@¤õ˜C"ą˜A"ņ˜C’ŖĀęÕĢ~˙ũwTø|‹-ĐĸE ŧōĘ+8}ú4Ž_ŋ^ã"ŠØĀ""""""""™ŠņŦ‰'VÚŧz˜zöėYĶSQųúú˛E$˜¯¯¯čˆ´sH$3H$sHr¤‘ģ^ģv ÉÉÉĒå#GŽ`ōäÉXētŠ&OU”œœĖ‘`eŋ ‰H æH,fH<æäH# ŦđđpėŪŊpãÆ ôėŲGŽÁ‡~ˆYŗfiâTwīŪe‹H°ģwīŠ.Hë1‡Db1ƒDâ1‡$Gi`:u mÛļŦ]ģžžž8pā~ũõWŦXąB§ *000`‹H°Ē^ZMDu‡9$‹$9$9ŌHĢ°°†††€;wbĀ€OOO¤ĻĻjâTl` (ē"­Į‰Å ‰Į’i¤åããƒÅ‹cßž}Øącúôé¸~ũ:5j¤‰SP9r„ ,"ÁŽ9"ē"­Į‰Å ‰Į’i¤õÅ_`ɒ% ÅĐĄCˆŽŽV]ZHõ„ ,"""""""’=M$44ˇnŨBVVŦ­­UëĮŽ Mœ‚ĒĀÎθråÁ 6°ˆęčˆ´sH$3H$sHr¤‘X IŽ;†%K–āŪŊ{”ĮąUĖĖĖęlVBBÖŽ]ûČm ˛ŗŗ+}~ņâÅ D`` lllāčč¨ZŪŊ{7–/_???čééáûīŋ/ˇ˙úõëáįįx{{#))ŠJĩģ¸¸ĀĶĶSuŽß~û­JûՄ™™™čˆ´sH$3H$sHr¤‘XWŽ\AŸ>}põęUäįįŖgĪž077Į—_~‰ŧŧ<,^ŧX§ĄĮ¸xņ"ęjÆą„„üųįŸ2dH1nÜ8Œ70räH„„„ā7ŪP=occƒĩk×âķĪ?/ˇo||<>ūøcėÚĩ ČĘʂž^Õ?žëÖ­ƒ¯¯ok'ĒĒ‹/rî?"Á˜C"ą˜A"ņ˜C’#ŒĀzë­ˇ‚;wîĀØØXĩūųįŸĮŽ]ģ4q ĒĒ*ŽĀÚ˛e‹jDR`` ŒŒŒ°råJÜŧyŊzõ‚ŸŸüũũ1jÔ(¤§§cúôéØšs'UM¨ 6ĀĶĶ:tĀėŲŗk]z@@ŧŧŧ ŖSūc9wî\ŧũöÛpppXXXpt‘–ĐČŦūųû÷ī‡Úzggg¤¤¤hâT^^^ĀɓV<ĸÕ¯_?ôë×°jÕ*Ė;aaaXļl\\\°}ûv@FFlll0kÖ,üųįŸXˇn ==cƌÁāáá/ŋü˛î^€3gΠE‹čÚĩ+˛˛˛đėŗĪ"""ēēēUÚØ°a())AģvíđųįŸŖI“&uZ/i////Ņ%i=æH,fH<æäH##°JJJP\\\n}rr2ĖÍÍ5q Ē‚´´´jĪĩ{÷nDDDā¯ŋū‚šš9Úˇo­[ˇâíˇßFtt4LMM+ÜīĐĄC ‚‡‡å„ũuŠ°°ĮŽÃÖ­[ą˙~>ļļļš8UApp0 P`ž‡ ’Đmũēu\\€ lģlŲ2ÜŧySĻLQXŠŽŽFLL ‚ƒƒˆN:áĢ¯ž‚ĨĨ%ēwīŽœœ`ܸq°ĩĩÅŌĨKŅŋtėØąÂ‰×ĢkÕĒUpttÄīŋ˙ŽiĶĻÁŅŅņņņ€—^z đņņA@@ėííÕî`X™´´4tëÖ ūūūđķķÞ={đķĪ?×ēVĸĘ‹.Hë1‡Db1ƒDâ1‡$G Š:ׂUbīŪŊčØącš.oQQ8PáH™†$++ –––ČĖĖ„………črjėȑ#HŪmÁīģAųK}ĐT*Ŋ˛pŨ: ,LDuDÚáȑ#hÛļ­č2ˆ´sH$3H$sH Qm{/Õ­[ˇ ¯ąÍĖĖDˇnŨ4q Ē‚ĸ" o}Ķŧ\ķ x0ÖäÉ@SZ‘†hā߈¨–˜C"ą˜A"ņ˜C’#4°$I‚ĸėäá˙š}ûvĨwą#ÍKJrBrē!*ûĩJpíĀ9Ė‰ęN“&MD—@¤õ˜C"ą˜A"ņ˜C’ŖZÍėößĩh …#GŽ„ĄĄĄęšââbœ8q;vŦ]…TeššU‚—šZĮ…i1kkkŅ%i=æH,fH<æä¨V#°,--aii I’`nnŽZļ´´„Ǝ‹UĢViĒVzŒ‚‚ĢUÚÎŪžŽ !ŌbįĪŸ]‘Öc‰Äb‰ÄcIŽj5+22āâ₩S§ōrAÁīÁŅļ)éz*čM*€Ŗ#ĐĨ‹€âˆˆˆˆˆˆˆˆjHcs`ŨēuK‡ĸZđööĀ‚wS ”¨=W:EŲüų€Žn=F¤E<<88EEEš8UAVV SæWĘIķ‰ę]VV–čˆ´sH$3H$sHr¤‘ÖđáÃąhŅĸrë—.]ŠaÆiâTiii€žūƒlÕģ´´4Ņ%i=æH,fH<æä¨Æ—ūß˙ũŸęąBĄĀ˛eË°}ûv´oßpčĐ!\ģv #FŒ¨}•TueXâę """""""Ō…$Õlĸ¤nŨēUí ū÷ŋ˙Õäõ&++ –––ČĖĖ„………črjįÖ- IåãgŸūøCl=DDDDDDD¤õjÛ{ŠņŦŨģw×tWĒ#qqqrs{°‚#°ˆę]\\‚‚‚D—A¤Õ˜C"ą˜A"ņ˜C’#ĖE CaaĄú%„……âŠ!ŌR…Ė‘pĖ!‘XĖ ‘xĖ!ÉQXãƍÃĩk×Ē´íoŋũ†ÕĢW×ôTTEÖÖրÁƒETīŦ­­E—@¤õ˜C"ą˜A"ņ˜C’Ŗ7°š4i___ôíÛ‹-ÂŅŖG‘’’‚ÛˇoãâŋˆŽŽÆģīž‹æ͛cūüųđ÷÷×dŨT@W÷ÁŠGtŨ“’’ P(āââR÷…•ĄP( P(Ēŧ}hh( bbb#FŒ€ Đ´iSŧüō˸{÷nŨKTĸK ŌzĖ!‘XĖ ‘xĖ!ÉQXŗgĪƅ đÔSOaņâÅhßž=š7o[[[xxx`Ĉ¸|ų2–-[†ƒÂĪĪO“uSNŸ> (Fa5ĀaŖQQQˆŠŠĒņū-Z´ĀŒ3°téR´hŅĢV­Â§Ÿ~ĒÁ ‰jįôéĶĸK ŌzĖ!‘XĖ ‘xĖ!ÉQ­æĀ˛ĩĩÅ|€ãĮãöíۈ‹‹Ãūũûņīŋ˙âΝ;XˇnzõęĨŠZŠĒJįÁĒÆ%„K–,ŽŽ:wîŒģwīâ‡~€››Œammž={âėŲŗ€˜˜( ´iĶŖF‚ĩĩ5<<ŌĶĶU˘:u* ::īŧķN•ĩpáBxyyÁ××ūūūČËËĢu}Ô°¸•Ŋ( Á‰Å ‰Į’é‰.€4'77Wų ôÂ*ŒĀJIIÁĉŅģwoDGGC˙ŋæ×íÛˇ1gΤĻĻĒļ‹‹SÛ×ÛÛ_|ņ.\¸€å˗ãâŋ€víÚ!$$/ŊôŌĶĶącĮ>ļ–7Ūx+WŽtíÚU5īÕÃ.\ˆëׯcņâÅØļm"##1yōäĮŋ2EEEĐĶĢ< ķįĪG=T͸˛ €Té<›7oÆęÕĢqčĐ!XZZ"==]õ~“|¨rHDÂ0‡Db1ƒDâ1‡$GE⩚Mz€ŸL -()ŽtcccXYYáĐĄC8~ü8å—Ũ„ pëÖ-,_žÛˇo‡ĄĄašŅBMš4UĻ¨¨ IFuëÖáíˇßƌ3ĒT˙ģīž‹;v`ĮŽ˜;wnĨÛĩmÛÄgŸ}ø駟*Ũöë¯ŋF§NāîîŽ6÷–BĄĀÜšsŠ>øiiixūųįáįį___,]ē0kÖ,\ŋ~ƒF`` ԎŋbÅ <¸J¯īĢ¯žÂĖ™3aii @y ŽnŲI÷IĘ6}‰H æH,fH<æäˆ#°äæÜj 08āÜĀģĐÛ˜žđVnsDEEĄgĪžčÕĢvîÜ ( !##@~~~•KøčŖ°bÅ xzz"$$kÖŦĢĢ+Úĩk÷ČũŧŊŊáíí]éķ§NÂ´iĶĐŖGaųō倀€€J÷Q(Øŋ?._žŒļmÛĸsįÎprräįįĢFyŊøâ‹đôôÄƍ‘žžŽāā`búôéXž|9Ö­[___ĀĻM›Ēü^”uæĖÄÆÆbƌČĪĪĮˆ#0iŌ¤‹ˆˆˆˆˆˆH›p–Œ„˜Ÿúū’€ ”ŽÉRŽ?ˇēÂũ:tč€ččhÜŋ={öą °páB4nÜß|ķ Ņ¨QŖ*×qāĀĀšsįT“ļ/Z´¨Ö¯ĪÜÜŠŠŠøđÃ1aÂ$''c„ øá‡*Ũ§tŪŽ-Z sįÎØˇoŸęšW_}Uõxį͘8q"¨æõÚĩkW­k.Ģ¨¨—.]ÂŪŊ{ą}ûvüøãØ˛e‹FĪA⅄„ˆ.Hë1‡Db1ƒDâ1‡$GUTT„˜˜\ēt ááá077Įõë×aaa333M†*SR iæxā ”Ģ˛$(×ĪšŦz ĐŅ…‹‹ $éÁ†O?ũ4îßŋ¯Z Ę1cTËe/ UÛ÷ácU6wĩíĒâác:t¨Zû?ŦėÍū\>|ãÚ܈āĖ™3tęÔ ?üđš7oŽĄC‡BWW666čÛˇ/Ž9‚~ũúÕø<Ôđœ:uę‘Ŗ‰¨î1‡Db1ƒDâ1‡$GuåĘøųųášįžÃĉqķæMĀ—_~Šē[Õą›û ˇ3§|ķĒ”`[ps_%<9Š‹˜ *JųgqåS|Š.3LJJÂ?˙üƒÎ;W¸]=Tķ^Ũŧy7nÄĶO? °°°xä]+âí퍄„$$$¨Fˆ…‡‡cëÖ­€ŧŧ<ėŲŗ‡˙Q‘!ŪY’H<æH,fH<æäH# ŦˇŪz !!!¸sįŒUëŸūy_†E•¸Ÿ ÜyĖ6wūÛî ļaāât넇+˙tqQŽ¯ˆĄĄ!:uę„^ŊzáģīžSÍõ°oŋũ'Nœ€ŋŋ?ēu놏>úmÛļLš4 ŖFĒp÷ę˜2e nܸoooŖoßžxūųįk|ūøcŒ=Z‡§*ĘÍÍU>q P^&*ē ĸŠŠrHDÂ0‡Db1ƒDâ1‡$G›ÄŊÔØącáîîŽÁƒcĪž=š><=‚™™™ōL/!$z¨rHDÂ0‡Db1ƒDâ1‡$Gš„ĐŲŲYm˛öĐĐP:tÉÉɚ8>—/_ÖÄ)¨ Ž?Ž|ĀXD¨rHDÂ0‡Db1ƒDâ1‡$Gi`UÆČČÎÎÎuy ĒG`‘ŒÔx,œ?7†ĩĩõ#ī6˜‘‘QĶĶP54oŪ\ų ė,6°ˆę•*‡D$ sH$3H$sHrTãÖŧyķ`nn˜?žĻę!M(;‹—ŅŽÆ ŦW^yĨÂĮ$ÎÕĢWaooĪK‰R合„a‰Äb‰ÄcIŽjÜĀ€’’”””@OīÁaŌŌŌ°xņbäää`Ā€čÜšs­‹¤jâ$îDDDDDDD$#ĩj`ŊöÚkĐ××ĮŌĨK÷îŨC›6m——{{{Ė›7›7oFŋ~ũ4R,=Z@@€ōG` ŖĘ! ÉŠ‰Į’Õę.„û÷īĮāÁƒUË?˙ü3ŠŠŠpáÂ?~˙÷˙‡¯žúĒÖERÕ\ētIų€#°ˆ„Q合„a‰Äb‰ÄcIŽjÕĀJIIAĢV­TËģví Aƒ`ii @97ÖéĶ§kW!UYvvļōG` ŖĘ! ÉŠ‰Į’ÕĒedd„û÷īĢ–:„öíÛĢ=ĪāÔå6°ˆ„Q合„a‰Äb‰ÄcIŽjÕĀ Ā/ŋüØˇoŌŌŌđôĶOĢžŋtéjW!U™§§§ō/!$F•C"†9$‹$9$9ĒUkÚ´i˜?>ÜÜÜĐģwoŒ9RíV7nD§Nj]$UM\\œōG` ŖĘ! ÉŠ‰Į’Õę.„ŨēuÃącĮ°cĮØŲŲá…^P{>00mÛļ­UTEDDDDDDD2RĢx{{ÃÛÛģÂįƎ[ÛÃS58::*p‘0Ē‘0Ė!‘XĖ ‘xĖ!ÉQ­.!¤†EOīŋ~dŲXl`Õ+U‰HæH,fH<æäˆ ,IJJR>(;‹—Õ+U‰HæH,fH<æäˆ ,9â%„DDDDDDD$#l`ɈŸŸŸōA=N➐€ĩk×>r…BėėėJŸ_ŧx18::Ē–wīŪå˗ÃĪĪzzzøūûīËíŋ~ũzøųųÁĮĮŪŪŪUū׆­[ˇ"$$ūūūhßž=Ž?^ĨũˆE•C"†9$‹$9$9bKFŽ^ŊĒ|ĐĀX3nÜ8$$$ !! Āûīŋ¯ZîÖ­‚ƒƒąvíZ„‡‡—Û7>>ü1ļmۆĶ§OãĐĄC°ĩĩ}ė9īÜšƒáÇã—_~Á‰'đÅ_`Ø°aĩzD@™‘0Ė!‘XĖ ‘xĖ!ÉQgvŗļļ†BĄ¨Ōļ5= UCffĻōĄáƒ•ųųÕ:Ɩ-[đá‡Ē–Ī;‡%K– _ŋ~6lRSSĄP(Œ/žøĶ§OGVVŅž}{,^ŧ6lĀ‡~kkkôëׯ֯+  ŖSžß:wî\ŧũöÛpppXXXT阗.]‚­­-ŧŧŧ]ģvŕ+W‡   Z×LÚK•C"†9$‹$9$9ĒqkūüųĒĮˇoßÆ'Ÿ|‚ŪŊ{ŖC‡€ƒbÛļm˜6mZ­‹¤Ē122R>¨ÅŦ~ũúŠšNĢV­ÂÜšs†e˖ÁÅÅÛˇo lJÚØØ`ÖŦYøķĪ?ąnŨ:@zz:ƌƒĀÃÃ_~ųeí_Ø#œ9s-Z´@׎]‘••…gŸ}ĐÕÕ}ä~­ZĩÂ͛7qčĐ!´oß7nDvv6’’’ØĀĸZQ合„a‰Äb‰ÄcIŽjÜĀzå•WT „Yŗfá7ŪP­›4iž˙ū{ėÜšSĻLŠ]•T%žžžĘĩUj÷îŨˆˆˆĀŪŊ{annŽöíÛcŪŧyxûíˇŅĩkWôîŨģÂũ:„   xxxƎ‹÷Ū{¯F5TEaa!Ž;†­[ˇB’$ 0K–,Á„ šŸĨĨ%Ö¯_÷ß÷îŨCįΝáíí ũ˛āՀ*‡D$ sH$3ø˙ėŨy\ÕUūĮņ×PdSwÅ 3MKËĨšqęg˨5Ĩ3•Ú^֌-ÖĖdVãXMļŽKcŲ4je™Vb–inäž&¨¸. îĀũũņ Čâ.øō~>ß÷~×s¯÷}Ņįœ¯ˆyĘĄØ‘GæĀúōË/šöÚk‹Ŧ2d_}õ•'.!nXģv­õ `Ŧr°6oŪĖčŅŖųôĶO]Cķz÷îMbb"ŊzõbūüųôėŲ“œœœ"Į:Nˇ¯Ķ§OâââčÕĢW™Û˜§yķæÜtĶMøųųQŋ~}† Ə?ūXdŋ÷Ū{Ī51üĖ™3¸ōĘ+IHH`Ũēuŧøâ‹|‡ÃAnn.ß˙=>ú(wÜq‡'.!nčŪŊ{ū“ŧĄœN'ÃnČaŪ<¸¸–ķæÁ°aU×Fģ+”C1B91K1O9;ōHëoû͚5ŖiĶĻdee˕W^IŸ>}xōÉ'=q qÃęÕĢķŸüŌ €sį6 ’’`Ų2øāëįŪŊ*^‰xZĄŠˆĘĄˆYĘ ˆyĘĄØQOœ¤nŨēŧ˙ūûüå/aũúõäææŌ­[7Ú´iã‰ĶKy\TĀĸAŧŊĄc-)°ō´lŲ’–-[’““ÃĻM›8~ü8ÁÁÁžŧ„”"<<<˙IģArū|Õ7F¤–*”C1B91K1O9;ōČÂ|ũë_äääpÕUWŅŊ{wĸŖŖIHHđÄ%Ä ųO.î%"UĸPEÄåPÄ,ePÄ<åPėČ#ŦyķæŅĩkW>ûė3~ūųgļoß΃>Čĉ=q qÃŽ]ģōŸ¨–ˆ…r("F(‡"f)ƒ"æ)‡bG)`Ĩ§§ĀĸE‹¸õÖ[iÛļ-øÃØ´i“'.!eĨX"""""""b)`…‡‡ŗuëVrrrXŧx1āôéĶx{{{ââ†:ä?QKĈB9#”Cŗ”Aķ”Cą#°FÍ­ˇŪJ§Np8 4°nŨŲž}{O\Bܐššš˙DCEŒ(”C1B91K1O9;ōHkŌ¤IŧûîģŒ3†īŋ˙Ÿ_z˙x{{ķÄOxââ†ŖGæ?Q,# åPDŒPEĖREĖSŎęxęD7ß|sĄį'NœāÎ;īôÔéÅ …†kĒ–ˆ6-bžr(b–2(bžr(vä‘XSĻLá?˙ųëų­ˇŪJŖFˆŠŠbãƍž¸„¸!>>>˙‰z`‰Q(‡"b„r(b–2(bžr(vä‘Ö[oŊEtt4K—.eéŌĨ|ņÅ\{íĩ<účŖž¸„¸a͚5ųOTĀ1ĸPEÄåPÄ,ePÄ<åPėČ#C:ä*`}ūųįÜzë­ <˜˜˜zõęå‰KˆrssķŸhĄˆ…r("F(‡"f)ƒ"æ)‡bGéĖūũûXŧx1Āét’““ã‰KˆBCCķŸ¨–ˆ…r("F(‡"f)ƒ"æ)‡bGé5lØ0FŽI›6m8zô(×]w‰‰‰´nŨÚ—7„„„ä?Q,# åPDŒPEĖREĖSŎ<Ōë˙øãĮ'66–ĨK—âīīXC ĮŽë‰Kˆvėؑ˙D=°ŠHLLäŖ>*u‡ÃAVVV‰Ûß|ķMââ∋‹#$$„¨¨(×ķe˖1cÆ :wîL:uøį?˙YäøųķįĶšsg:vėHll,IIInĩũūûī'&&‡ÃÁæ͛ m=z4]ēt!..Žž={ōõ×_ģuNŠ…r("F(‡"f)ƒ"æ)‡bGéUˇnŨb'kđÁ=qz)°ŠHLLtÍŅV^÷Üs÷ÜsŖF">>žņãĮģļ‡„„đŅG1yōä"ĮnØ°'Ÿ|’¯ŋūšČČH233ŠSĮŊŪ|ķÍL˜0ž}ûŲöüƒ   Āz$-- ‡ÃQŽW(""""""Rũx¤Āŋ˙ũoúöíKdd$ÉÉÉL›6O?ũÔS—KhÛļmū“Z0„pŅĸEŽŪOqqqøúú2{ölŌŌŌÄÅÅŅĢW¯rŧKķæÍšéĻ›đķķŖ~ũú 6ŒüąČ~īŊ÷žkXäĖ™3ËtrōäIŽ Rī7ķ”Cŗ”Aķ”Cą#°Z´hAbbb‘õ_|ņąąąž¸„¸á˛Ë.ËR°V +`-X110`ŒiũŒ‰ąÖ”’’7ŪČĖ™3éØąŖkũŪŊ{ņ÷÷įÖ[oåĩ×^cįΝdee@FF†kŋŪŊ{ŗaÃvîÜ Āģīž[b›VŽ\Ibb"ĢW¯.÷ë9r$K–,!77—œœ–.]J׎]‹ėwĮw¸†FŽ=ēÔsfggŗk×.×ķü‘ÔÔTZļlYîvJÅĘĄˆĄŠ˜Ĩ Š˜§Šy¤€õØc1nÜ8ūķŸ˙āt:ųņĮųÛßūƟ˙üg{ėą2ë7Ū K—.@īŪŊųâ‹/J=fųōåôčŅ___ZļlɛožY‘—Sc­[ˇ.˙IÁX5h8Ų‚pķÍpā@áõ))Öú‚EŦwß}—´´4zč!WĨ… ’@=ˆ‹‹ãŠ+ŽāĨ—^"00kŽš†S§NŅĩkWîšįÂÂÂxûíˇ:t(}úô)vâõ˛š3gQQQü÷ŋ˙劧ž"**Š 60|øp"##éØą#]ģvĨI“&…î`XšqãÆŁ8p ­[ˇ ''‡QŖFŅŠS'ēvíʃ>ČŧyķŽđk‘ō)”C1B91K1O9;r8Ë2ŽĒīŧķũë_Ųŋ?M›6eŌ¤Iüá(Ķy>ûė3ŧŊŊ]˙@Ÿ={6/Ŋô6l(ÔË&ĪŪŊ{éÔŠüãšûîģųūûī;v,sįÎåĻ›nr뚙™™’‘‘A@@@™Ú[Ŧ^Ŋ:ˆÛŽ]wį‰Ûnƒ9sĖ5ĖM99VOĢ‹‹WyˆŠ‚Ŋ{ĄĀTk"ÕJĄŠˆĘĄˆYĘ ˆyĘĄTG­ŊT¸ËIvv6ŗgĪfčĐĄ$''“ššĘáÇŲŋ™‹WC‡åú믧mÛļ´mۖŋũíoøûûŗjÕĒb÷ķÍ7iÖŦĶĻMŖC‡Üu×]üū÷ŋįå—_ŽčKĢqBBBōŸÔ¯Ÿ˙øĖ™ĒoL9ŦXQrņ Āé„ũû­ũDĒĢB9,‡ÃAVVV…Û1mÚ4RSS]Ī'MšÄŖ> Ā… Ũîģk×. D׎]éØą#˙ųĪ*Ü6‘ĘVŅŠHÅ(ƒ"æ)‡bG.`ÕŠS‡{īŊ—sŋĖŗÔ¸qcÂÂÂ*Ü0°†G}øᇜ:uŠŪŊ{ģĪ?üĀāÁƒ ­2dk׎å… ÅsîÜ9233 -vž˙¤`Ģ†L¤čg÷1ĄP+QvvvŠÛ/.`ô›ßü†—^zÉ­ëŒ5ŠÛnģŸ~ú‰ožų†Į{Œ”””2ˇW¤*UUE¤xĘ ˆyĘĄØQOœ¤W¯^lØ°æ͛{âtlÚ´‰ŪŊ{söėYüũũųøãKœ ūđáÃEÂNvv6ééé4iŌ¤Č1“'OæŲgŸ-˛~íÚĩ4hЀîŨģŗmÛ6Μ9CÆ iŅĸ7nŦ;Éåææē†JÆÅÅą{÷n˛˛˛hĐ mÛļuÍy…ˇˇ7ÉÉÉtéŌ…¤¤$233ņõõĨcĮŽŽąÉ‘‘‘øúúōķĪ?ĐŠS'8Ā‰'¨W¯qqqŽ;ÖEDDāīīĪîŨģčĐĄGŽaĪž=„††ŌŖGÖlŨJĪ_^ׅĖLÖ˙2ųxģví8vėiiixyyŅŗgOÖŽ]KNN5",,ŒmÛļĐĻM2339rä`ũ9¯_ŋž .Ldd$[ļl UĢVœ>}šCŋT—âããŲŧy3gĪž%00f͚šîŒCvv6~énÕŊ{wļoßÎņãŪĀĨ'ũ?{v/ĢW§Rŋ~}ÚˇoĪúõë]īw:uHJJ sįÎėÛˇŒŒ |}}éÔŠk׎ I“&Ô¯_Ÿ={öĐącG<ČņãĮŠ[ˇ.ŨģwwMØN@@€k˛ô:ššĘŅŖGņöö&>>ž5k֐››Khh(!!!ėØą€ļmÛrüøqŌŌŌp8\vŲeŦ[ˇŽėėlBBBwŊß­[ˇ&++‹Ã‡Öċ‰‰‰œ?ž   ĸĸĸØŧy3-[ļäėŲŗŗ999Ž÷ģ[ˇnėÜš“S§NáīīOëÖ­]7aˆŽŽÆËËĢĐgvīŪŊœø k׎åôéĶüîwŋcđāÁDFFŌŦY3îģī>žûî;ŽŧōJnšå&NœHJJ >>> :”oŧ‘˙üį?ēvíʞ={ČĘĘŌwD-ūŽøᇠĒ–ßÕųīĮŽŖN:ôčŅƒü§ĶyÉīˆęô÷}GTŸīˆÄÄD‚ƒƒĢíwDuū{„ž#ôáŠīˆS§N_-ŋ#Ēûß#ôQyßÅÕgĘÂ#s`ũ÷ŋ˙å‰'žāĄ‡ĸG4hĐ Đöŧ”šëüųķėÛˇ'N0ū|Ū}÷]–/_^lĢmÛļŒ=š?ũéOŽuß˙=}ûöåĐĄCDDD9æÜšsŽc`ÃŒŽŽļ×XN'Ô­kM,ÕŖüōeZǻ•’b5˙bšKj‚ŠÎ7āp8˜4iĪ<ķ ?˙ü3—]v6l ::‡ÃáēAĀoû[ZļlÉäɓIMMĨGĖŸ?ŸË.ģŒ˜˜>˙üs:uęXCŗ˛˛xų嗙5kŸū9ķæÍģd{ ĀĐĄCyøá‡Ųŗg—]vˇŨv¯žújš_ŖHeĶŧ"f)ƒ"æ)‡RUt,ôĀúío Āũ÷ßīZįp8p:8rrrĘtžzõęš&qĪĢ8ŋōĘ+ŧõÖ[EöˆˆpU5ķ¤ĻĻR§N5jTėų}||đ)x—>›Č{ĪĢÚSŋ>œ]â1III8bbb*¯aÅp88ōĒR ķæAĶĻ…××ĢקĶAHHBĄõ÷ßŋë\y];ELōÄė+˜˙ˇ÷ŧ,ļnŨJ\\qqqŒ7°ē)˙÷ŋ˙%11‘ ‘‘Qân‘ęĸ2r("îSEĖSŎūøcŽžújČČČ(SÛcccILL$11‘×_€#GŽ7ŌüË/ŋdëÖ­ŽŪ`"ÕUI9‘ĒĄ Š˜§Šy¤€°cĮƏĪ5×\ÃĀ?~ŧkrš˛8räŋûŨīh׎×\s ĢW¯fņâÅ 4€C‡š&ŲkÂŗE‹‘@\\ųË_xõÕWšéĻ›<õŌj.7z`]ė­ˇŪÂËˋž}ûrâÄ ^ũuZĩj…ŸŸÁÁÁ 4Č5`BB‡ƒž={2zôh‚ƒƒi׎k‚ŧQŖFšzGå-ŖF`ĈŒ1ĸÄvx{C˙ū0b„õķâN%ĮŽcôčŅL˜0AwؐjoÁk~ˇ`äHëgLŒĩž8>>>\qÅ <˜×^{čččb÷{õÕWŲ¸q#]ētaĀ€Lœ8Ņ5|đūûīgôčŅÄÅÅš&Î,Ī>ûŒ6mÚĐž}{^xá-Z„ŸŸ_šĪ'"""""R™Ä}ŪŧyŒ1‚øøxz÷î ĀĒUĢXŗf |đˇÜrK…Z™*:‘Xu‘7į˜Ëå—Ã/Å$rrĀĢhŊ2))‰-ZĐŧys&L˜Āøņã2dķįĪ§~ũúĖŸ?Ÿ””|}}INNfʔ)ôęՋīŋ˙ž„„ Ā„ HOOgƌ\uÕU$$$°zõjöîŨËųķįyėąĮHMMå­ˇŪbĖ˜1Žv÷ņËĘĘâėŲŗÔ­[—ĀĀ@ú÷īĪōåËYļlũû÷į–[n!))‰•+WŌĻM’““ŲļmíÛˇ÷đģ*R6įpÁk^ˇ‹?ęyģĖ›gõ:Ī)ōûPDĒ”2(bžr(ÕQEk/é5aÂūô§?ņÃ?0uęTĻNĘʕ+ųķŸ˙Ėã?î‰KˆŠô˛ČBpæLŠĮĻ¤¤0nÜ8Ė… Š˙ËąGå…^āîģīæųįŸ'''ĮukĶ<ąąąL™2…'žxĀu;Í^ŊzqË-ˇđŲgŸ‘ššĘ¤I“3fĖ%_Įøņã %44”n¸ĄČöO>ų„?ū˜gŸ}–ääd˛ŗŗëöˇ§ËĐÛL¤2ĖaN<đ@ņwÕĖ[÷āƒĨ'‘˛ĢH¯CŠ8ePÄ<åPėČ#ŦÇš—*ĪíˇßŽąˇUčüųķ…Wøׇ@oāĀRČ-ų_É~~~ąjÕ*×]ÍNŸ>ÍØąc]=Ģ–,Y‚ĢwTžĐĐPפųy%§ĶÉ]wŨÅŧyķxä‘Gxæ™gÜz&L`éŌĨ,]ē”ŋ˙ũīEļ'%%‘““ï~õ+Ú´iCJJ C† á›ožqë"•Ĩ`WŦ€JŪ×é„ũû­ũDÄsŠü>‘*Ĩ Š˜§ŠÕņÄIú÷īĪŠ+ŠÜĒķģīžŖ_ŋ~ž¸„¸!(((˙Éö÷ÁëK8lúü €§§CûۊÂÜšs4hƒæĢ¯žĸ]ģv8˛ŗŗ9vė+WŽ,ņnÅ™8q"ŗfÍĸ}ûöÄĮĮķá‡ŌĸE z7#{ąąąĨŪåė×ŋū5QQQŽįcĮŽ%--W^y…øøxˇÛ'R æđĐ!÷Žqw?qOĄß‡"Rå”Aķ”Cą#°~ķ›ßđøãŗnŨ:.ŋürĀšëŋ˙ũ/Ī>û, .,´¯TWQgûûpŨí ä ]:|˜ ?Ü_PlĢwīŪ,\¸_ũęW 4ˆ¯ŋūšéĶ§3qâDĻNĘ˜1chÔ¨GuĢ=+WŽ´šŗ}ģkÂö;īŧķ’ŦKiŨēuĄbiŪ'LDDD…Î-RQ‹ĢMš¸wŒģû‰ˆ{ æPDĒž2(bžr(vä‘IÜŊŠ™ŧ؋9äTÃÉ^ė2‰ûęÕĢéÕ3nąŠUÅũÉ:€á0įxyWuEloõęÕŽ"mNŽuˇÁ””âįÁr8 * öîĩîŧ)"žQ0‡"Rõ”Aķ”CŠŽĒÅ$îšššn-Õąxe;i+āËŠW`­˙2ĶÚOD*•ˇ7ŧōŠõøâ›Āä=Ÿ6MÅ+‘KņHKĒ‡–-[™Cpü;ĮÚOD<Že˖…žķæAĶĻ…÷‹Š˛ÖV…Š%.ÎĄˆT-ePÄ<åPėČ#s`üøã$$$ššJnnnĄmS§NõÔe¤gĪžŋ&Œ5įUI‚ąöģø.`Šn¸ÁēÛāĄC֜WũúŠį•He).‡"Ru”Aķ”Cą#°žūyž|ōIÚĩkGxx8Žce›‘JsđāAĸ{öŗî6XÚXC Tw‡Š $::ēČzoočßŋęÛ#R•”CŠĘ ˆyĘĄØ‘G X¯ŧō 3fĖ`Ô¨Qž8T„—7<=ŨēÛ`Áģ‚UŧjŽĩ]¸‹ˆˆˆˆˆˆH á‘9°ŧŧŧ¸âŠ+J§Nhßž=øÃ8ūŧÛmĖãĘĄˆŖŠ˜Ĩ Š˜§Šy¤€õĐCņúë¯{âTR[ļlÉŌū6¸c.„—OŒ†9ĮTŧ’)22’e˖šĩī˜1cØąc‰‰‰üú×ŋfĖ˜1n÷¯ũ‹7˛~ũzļmÛXŊKËĒPEÄåPÄ,ePÄ<åPėČ#ŦG}”;vĐĒU+†Ę°aà -R5ŠLÔįۀ€Ŗa6(ÕĘË/ŋĖW\AÛļm™;w.‹-"..Îĩøúú2{öl’’’hܸąëX‡ÃÁ¤I“ŠīëëËõ×_īš{īōË/įįŸvĢ=?ũô¤^Ŋz8Žŋūzūũī—ųuiÂLķ”Cŗ”Aķ”Cą#°îģī>–-[FÛļmiÔ¨…Š…WÔ¯Ÿ˙øôéĒmŒČ%8ž˙ū{/^Ė}÷ŨĮūũûšūúëILL$11‘G}”:”X/îø‹ŊúęĢ :Ô­öôėŲ“O?ũ”“'Orūüy>üđCˇ‡T$‡"Rå”Cŗ”Aķ”Cą#LâūŪ{ī1ū|~õĢ_yâtRN111…W4h˙X,Šfîēë.ZļlIßž}Yąb#GŽ`Ų˛eLš4‰oŋũ–† rôčŅ2ÖŨQwíÚśožéV{î¸ã’““šōĘ+iĐ ä›ož)ķë*’CŠrĘĄˆYĘ ˆyĘĄØ‘Gz`…„„ĐĒU+OœJ*`ãƍ…W,`ŋ äîģīfęÔŠš„”ArrrŅ•B(RĨŠÍĄˆT)åPÄ,ePÄ<åPė¨BŦŸ~ú‰k¯ŊļÄíƒfŨēuš„T”†ŠˆˆˆˆˆˆH WĄ֑#G¨[ˇn‰ÛëÔŠCZZZE.!eĐĨK—ĸ+5„P¤J›CŠRĘĄˆYĘ ˆyĘĄØQ… XM›6eĶĻM%n߸q#Mš4ŠČ%¤ öîŨ[tĨ†ŠTŠbs("UJ91K1O9;ĒPëúë¯į駟æėŲŗEļ9s†gžy†_˙úך„”ÁÉâ†æ?ÎȨēƈÔRÅæPDĒ”r(b–2(bžr(vT§"?ųä“,X°€ļmÛ2~üxÚĩk‡Ãá`Ûļmŧūúëäää0qâDOĩU.ÁĪĪ¯čĘāāüĮĮW]cDjŠbs("UJ91K1O9;r8NgENœœĖŊ÷Ū˗_~IŪŠC† aúôéÄÄÄxĸ•*33“ĀĀ@222ŠŊŖbMqá…ĸs’9P¯dgC\lØ`¤m"ĩEą9‘*ĨŠ˜Ĩ Š˜§JuTŅÚK…†4oŪœE‹‘žžÎęÕĢYĩjééé,Z´¨F¯ėdũúõEW:ųŊ°ÔK¤Ō›CŠRĘĄˆYĘ ˆyĘĄØQ…†LĪž==u:ņ¤ā`HKƒ'LˇDDDDDDDD¤Ė*ÜKĒĻM›ŋ!¯VFääT]ƒDjĄs("UF91K1O9;RËFęÕĢWü†  üĮēĄHĨ*1‡"Re”Cŗ”Aķ”Cą#°ldīŪŊÅo(x'B #ŠT%æPDĒŒr(b–2(bžr(v¤VmP°€Ĩ‰ÜEDDDDDD¤†QËF:wî\ü°DĒL‰9‘*ŖŠ˜Ĩ Š˜§ŠŠ€e#û÷ī/~CÁ9°4„P¤R•˜CŠ2ĘĄˆYĘ ˆyĘĄØ‘ X6rĸ¤â”z`‰T™s("UF91K1O9;RËF|||Šß –H•)1‡"Re”Cŗ”Aķ”Cą#°l¤K—.Åo(8„P,‘JUbE¤Ę(‡"f)ƒ"æ)‡bG*`ŲȚ5kŠßP°–ē’ŠTĒs("UF91K1O9;RĢ6 É|ô¨švˆˆˆˆˆˆˆˆ”ƒ X6Qü†Æķ§§WMcDjŠs("UF91K1O9;RËF4hPŌđõĩ§ĨU]ƒDjĄs("UF91K1O9;RËFöėŲSü‡BC­Į*`‰TĒs("UF91K1O9;RĢļČ+`Ĩ§ƒĶiļ-""""""""e –ÄÆƖŧ1oŦœŨ‰P¤•šCŠĘĄˆYĘ ˆyĘĄØ‘ X6røđá’7æõĀ #ŠDĨæPDĒ„r(b–2(bžr(v¤–;vŦä*`‰T‰Rs("UB91K1O9;RËFęÖ­[ōƂŦôôĘoŒH-UjE¤J(‡"f)ƒ"æ)‡bG*`ŲH÷îŨKŪ˜7¨–H%*5‡"R%”Cŗ”Aķ”Cą#°ldõęÕ%oÔB‘*QjE¤J(‡"f)ƒ"æ)‡bG*`Õ*`‰ˆˆˆˆˆˆH Ĩ–„……•ŧQs`‰T‰Rs("UB91K1O9;RËFKŪ¨9°DĒDŠ9‘*ĄŠ˜Ĩ Š˜§ŠŠ€e#ģví*ycp0x{[UĀŠ4ĨæPDĒ„r(b–2(bžr(v¤VmáåYUĀ‘D,ißž}é;ä̓Ĩ9°D*Í%s("•N91K1O9;RËFŌ/U˜Ę+`>m-"âq—ĖĄˆT:åPÄ,ePÄ<åPėH,šä—”&rŠtú˂ˆyĘĄˆYĘ ˆyĘĄØ‘ X6âåu‰?ÎŧX –H%šdĨÂųčŖJŨĮáp••Uâö7ß|“¸¸8âââ !**Ęõ|Ų˛eĖ˜1ƒÎ;S§NūųĪ9~ūüųtîܙŽ;KRR’[mŋ˙ūû‰‰‰Ááp°yķæBÛÖŦYÃW\A—.]ˆ‹‹ã›ožqëœR”r(b–2(bžr(vTĮtÄszöėYú XĒȋTŠKæP*,11‘Ī?˙œ[oŊĩÜį¸įž{¸įž{5jņņņŒ?Ūĩ=$$„>úˆÉ“'9vÆ <ųä“|ũõ×DFF’™™I:îũ:Ŋų曙0a}ûö-´Ūétō˙÷üûß˙fĀ€lßžAƒąsįNüüüĘũ:k+åPÄ,ePÄ<åPėHeYYŗfMé;¨–HĨģdĨˆE‹šz?ÅÅÅáëëËėŲŗIKKcđāÁtîܙ.]ē0zôhRSSyúé§ųęĢ¯ˆ‹‹sĄ,X@ûöíéŨģ7ųË_*ÜĻŽ]ģŌĄC‡b˙÷ōī˙;<ō‘‘‘Pŋ~}ˇÎ{å•WUdũŅŖG9vė Ŧ‰Wƒ‚‚øâ‹/*đ*j/åPÄ,ePÄ<åPėH,ÉÍÍ-}́%Ré.™C)âúë¯'11‘ÄÄD}ôQ:tčĀ°aØ3g111lÚ´‰7ō÷ŋ˙°°0ž{î9Hbb"ožų&ŠŠŠüņäĶO?å‡~ĀĮĮ§RÛģuëVöíÛĮUW]EˇnŨxꊧČÉÉŠĐ97nLxx8ķįĪ`õęÕėÜšĶíĄ‰R˜r(b–2(bžr(v¤–4.X *Žz`‰TēKæPJ´lŲ2&MšÄ˙ū÷?6lČå—_Îâŋyä‘GX¸p! 4(ö¸UĢVŅŊ{wÚĩkĀ˜1c*ĩ.\`Ũēu,^ŧ˜īŋ˙ž~øˇŪzĢÂįũôĶOy÷ŨwéŪŊ;Ķ§O§oßžÔ­[×-Ž}”Cŗ”Aķ”Cą#°lD,ķô—…ōŲŧy3ŖGæĶO?u ÍëŨģ7‰‰‰ôęՋųķįĶŗgĪb{:9Nˇ¯Ķ§OâââčÕĢWšÛÚŧysnēé&üüü¨_ŋ>Æ ãĮ,˛ß{īŊį9sæĖKžˇK—.|ņÅŦ_ŋžŲŗgsđāAbccËŨÎÚL91K1O9;RËFļoß^úaaųSS+ˇ1"ĩÔ%sXËää@BĖkũ,n¤]JJ 7Ūx#3gΤcĮŽŽõ{÷îÅßߟ[oŊ•×^{;w’••E@@Žũz÷î͆ Øšs'īžûn‰íYšr%‰‰‰Ŧ^Ŋēܯiäȑ,Y˛„ÜÜ\rrrXēt)]ģv-˛ßwÜá9zôčKž÷đáÃŽĮīŧķ 4āęĢ¯.w;k3åPÄ,ePÄ<åPėHŦÚ¤qcp8ŦĮ*`‰H%[°bb`Ā9Ōúc­/čŨwß%--‡zČÕciá…$$$ĐŖGâââ¸âŠ+xéĨ— äškŽáÔŠStíڕ{°°0Ū~ûm†JŸ>}ûė3>ūøcyßŲ"""""ĩœÃY–ą6•™™I`` ˜nNš;vŒŌw …ôtë_‘{÷VIģDjˇrX ,X7ß ˙†ÉĢĮĖ›ÆU}ģ¤vPEĖREĖSĨ:ĒhíE=°l¤āš…‡[?)ú/KŠ0ˇrhs99đĀÅÅä­{đÁ⇊x‚r(b–2(bžr(v¤–¤ē3,0¯€uæ deUnƒDj!ˇrhs+VĀ%ow:a˙~k?‘Ę Š˜Ĩ Š˜§ŠŠ€UÛh"wŠd‡yv?°lÄ­ÛÂįõĀka ’’’p8ÄÄÄTŧaeāp8Ę4iq˙ūũq8$$$z^p™6mZå4V¤nåĐæš4ņė~"eĨŠ˜Ĩ Š˜§ŠÕ1Ũņœõë×ĶŊ{÷Ōwrŗ€eĘÜšs+|ŽÆķÚk¯šž_ō=ņ ˇrhsũúAT¤¤?–ÃamīׯęÛ&ĩƒr(b–2(bžr(v¤X6rá…KīTpa XoŊõ^^^ôíۗ'Nđúë¯ĶĒU+üüüfĐ AlÛļ €„„={ödôčŅĶŽ];V¯^ Ā¨QŖŠô’5j#FŒ`ĈnˇĢ8 4ā×ŋū5ˇÜr ǧmÛļ:ŸHY¸•C›ķö†W^ą_ÜĄ2īų´iÖ~"•A91K1O9;RËFÜēMjÁXn΁5}útîŊ÷^† ’%K """‚x€W^y…ącĮ˛lŲ2îēëŽBĮ­]ģ–°°0† ÆΝ;yüņĮ¸÷Ū{™;w.ŗgĪ&ė—‚ZŸ>}.ŲŽŦŦ,ŌĶĶIOO/õŽûöíŖaÆøúúrÍ5×°{÷nˇ^§ˆ'čvŖaÃ`ŪüđCŪ˙}FÅwß}wÉķ‹x‚[9Ŧ%† ƒn°î6xč5įUŋ~ęy%•O91K1O9;RËFļnŨzéÉúÂÃÁ´ꮅ# ÚŧŠ˙ĨŸŸŪŪŪŦZĩŠŸ~ú‰øøxNŸ>ÍØącņōōbƌDEE1tčPΞ=[čØĐĐPWŅ+;;§ĶÉ]wŨÅŧyķxä‘Gxæ™gÜz}&LāöÛo 88¸Ø}xā×ãŪŊ{ķūûīŗqãFˇÎ/â nå°ņö†ūũMˇBjåPÄ,ePÄ<åPėHŦÚæÄ7p° xõGø÷OO‡öˇŲ=$$„šsį2hĐ ĖW_}Eģvíp8dggsėØ1VŽ\ÉšsįÜnÂĉ™5kíÛˇ'>>ž?ü-Z\ō 666–ØØØˇgddpŨu×qã7ÆüųķčÛˇ¯Ûm‘ęGs`ŲHĢV­JßaûûpÃhø88ųųa&\wģĩŊŊ{÷fá…œ9s†Aƒąk×.ĻOŸNãƍ™:u*QQQ4jÔČívŽ\šŌjÎöíŽIÛßxã ˇ/‰ŅŅŅŧúęĢÜ}÷ŨŦ[ˇŽŅŖGģ†ŠT…KæPD*r(b–2(bžr(väp:‹ģÉy풙™I`` ˜nNš%''Ķŧyķâ7ææĀí!VąĒ¸?q0<æ+q8Ąˆ\ZŠ9‘*ĄŠ˜Ĩ Š˜§JuTŅڋz`ŲČáÇKŪ˜ļž,ĄxÖú/3­ũD¤ÜJÍĄˆT åPÄ,ePÄ<åPėHŦÚâĖ!8~‰}Ž˙˛ŸˆˆˆˆˆˆˆH5ĸ!„Øgann.^^%Ô$$@ėkÎĢ’„[—AxĪ7N¤–(5‡"R%”Cŗ”Aķ”CŠŽ4„P\6nÜXōÆĐ~ÖŨ%lw`míWMŠ5JÍĄˆT åPÄ,ePÄ<åPėH,9wî\ÉŊŧáééМĸE,Öú§§kw‘ *5‡"R%”Cŗ”Aķ”Cą#°l$((¨ôÚß_Ė›üŦá‚ŦŸÃŦõíoĢüFŠØÜ%s("•N91K1O9;Ēcēâ9ŅŅŅ—ŪŠũmđ@4lš Úo‚I˙QĪ+q+‡"RŠ”Cŗ”Aķ”Cą#õĀ˛‘M›6šˇcD$l~v×SņJăÜÎĄˆTåPÄ,ePÄ<åPėHŦÚ(,,˙ņ‘#æÚ!"""""""â°l¤E‹îíØ°!øúZUĀņ(ˇs("•F91K1O9;RËFΟ?īŪŽ„‡[SS+¯A"ĩÛ9‘JŖŠ˜Ĩ Š˜§ŠŠ€e#)))îīœWĀJO‡ėėĘiH-TĻŠHĨPEĖREĖSŎTĀĒ­ōæÁr:­"–ˆˆˆˆˆˆˆH5Ĩ–tīŪŨũķz`æÁņ 2åPD*…r(b–2(bžr(v¤–lÛļÍũ °4–ˆĮ”)‡"R)”Cŗ”Aķ”Cą#°läĖ™3îīŦX"•ĸL9‘JĄŠ˜Ĩ Š˜§ŠŠ€e# 6tįŧ9°@,*SE¤R(‡"f)ƒ"æ)‡bG*`ŲH‹-ÜßY=°D*E™r("•B91K1O9;RËF6nÜčūΚK¤R”)‡"R)”Cŗ”Aķ”Cą#°j+õĀ‘B,iŪŧšû;ƒˇˇõX,)SE¤R(‡"f)ƒ"æ)‡bG*`ŲHnnŽû;{yåO䎖ˆĮ”)‡"R)”Cŗ”Aķ”Cą#°ld˙ūũe; oaj*8žoH-T抈Į)‡"f)ƒ"æ)‡bG*`ÕfyŦėl8~Ül[DDDDDDDDJ –ÄÅŕí€ŧ!„ a„"R抈Į)‡"f)ƒ"æ)‡bG*`ŲČîŨģËv€îD(âqeÎĄˆxœr(b–2(bžr(v¤–dee•í€‚ŦÔTĪ6F¤–*sEÄã”Cŗ”Aķ”Cą#°l¤Aƒe;@CE<ŽĖ9SEĖREĖSŎTĀ˛‘ļmۖí !ņ¸2įPDtČ\;DDDDDDD¤ĖFŒÁ˛eË2d_ũ57.´ũŪ{ī%22’§žzę’į dūüų<ņÄœŋŋ?­ZĩrŨĻŗYŗfėÛˇ€Ž]ģ˛gĪ˛˛˛¨_ŋ>íÛˇwM&E:u\cs;wîĖž}ûČČČĀ×חN:ąvíZĀ*zÖ¯_Ÿ={öÖ$‡äøņãÔ­[—îŨģŗzõjÂÃà `׎]Ž÷;55•ŖGâííM||híWÛTģÖ¸qã˜3g|đ 6äđáÃ>|˜3gθöųĶŸūThŒæ´iĶøä“OØĩk[ļláOúķįĪgüøņ&^‚1yŨ6ËL,)wEÄc”Cŗ”Aķ”ÚkÅ (íĪé„ũû­ũj›j7‰ûoŧ@˙ūũ ­Ÿ9s&ŖFŦ ÅōÆ)œ?žG}”””üüüčØą#˙ûß˙¸ūúëĢĒŲÕ‰'Ęw  X"SĮ(‡"f)ƒ"æ)‡5×/SŊyl?;Šv,wĻäš5kVĄį&L`„ •ÔĸšŖ^Ŋzå;P,)wEÄc”Cŗ”Aķ”ÚËŨ)ŠkãÔÕÕnĄ”_\\\ųTKÄcʝCņåPÄ,ePÄ<å°æę×ĸĸō'lŋ˜ÃŅŅÖ~ĩ X6’wëÛ2+XĀúåVŠ"R>åÎĄˆxŒr(b–2(bžrXsy{Ã+¯X/.bå=Ÿ6ÍÚ¯ļQK (üüŦĮę%""""""bĖ°a0o^áž&`õĖš7ĪÚ^Uģ9°¤ü"""Ęw Ãa%c÷n°D*¨Ü9QEĖREĖSkžaÃā†Ŧģ :dÍyÕ¯_íėy•G,ņ÷÷/˙ÁyŦŒ 8u 4đ\ÃDj‘ åPDė­ˇŪÂËˋž}ûrâÄ ^ũuZĩj…ŸŸÁÁÁ 4ˆmÛļ€Ãá gĪžŒ=šāā`ÚĩkĮęÕĢ5j”Ģ§UŪ2jÔ(FŒÁˆ#Ęũōūú×ŋđŋ˙ũŅŖG3~üx^{íĩrŸO¤$ĘĄˆx„r(b–2(bžr(v¤–üøãå?¸Nü^Xn°ĻOŸÎŊ÷Ūː!CX˛d AAADDDđĀđĘ+¯0vėX–-[Æ]wŨUč¸ĩk×Æ°aÃØšs'?ū8÷Ū{/sįÎeöėŲ„……ЧOŸKļ#++‹ôôtŌĶĶÉČČ(˛=--}ûöáããï~õ+4h@ãƍyã7Üz"eQĄŠˆG(‡"f)ƒ"æ)‡bG*`ŲˆĶéŦØ š7ˇ~= §N•ēkJJ ãƍcđāÁ,\¸úõë˙rčQ^xáîžûnžūyrrrXŋ~}Ącccc™2e O<ņģwī W¯^ÜrË-|öŲg¤ĻĻ2iŌ$ƌsÉf?žĐĐPBCCšá†Šl÷ūĨgŲšsį¸æškøčŖđööfܸqlÚ´é’į)‹ įPD*L91K1O9;RËFBCC+v‚ŧ\˛–ŸŸAAAŦZĩŠŸ~ú €Ķ§O3vėXŌĶĶ™1cK–,ÁĮĮ‡ŗgĪÛÎēu됝 X_˛wŨuķæÍã‘Gá™gžqĢŲ&L`éŌĨ,]ē”ŋ˙ũīEļ‡„„Āũ÷ßĪ-ˇÜBŋ~ũp:ėÚĩË­kˆ¸ĢÂ9‘ SEĖREĖSŎTĀ˛‘āāāŠ y4tzģ?ƒÜœw aŅĸEdgg3xđ`Ö¯_ĶéÄápÍącĮøčŖ8wîœÛ—Ÿ8q"ŗfÍĸ}ûöÄĮĮķá‡ēæĮ*Mll,dāĀ%Žõ7nĪ=÷ožų&_ũ5ūūū\~ųånˇOÄÎĄˆT˜r(b–2(bžr(v¤–ėÜšŗüoļM…#Ā*`ôp{ˆĩžŊ{÷fá…œ9s†Aƒąk×.ĻOŸNãƍ™:u*QQQ4jÔČí&Ŧ\šŌjĘöíŽIÛ=5OՓO>É<ĀŌĨKyä‘GčĐĄŸū9‘‘‘9ŋHž åPDíš“Š€%R.ÍĄˆ”‹r(b–2(bžr(v¤–:tČs'SK¤\<šC)åPÄ,ePÄ<åPėH,)^LLūãŊ{5CDDDDDDDD,‰÷ÜÉĸĸ Nëq5)`%%%áp8ˆ)X\Ģ‡‡Ãáöūũû÷Įápājsq‹Ø“Gs("åĸŠ˜Ĩ Š˜§ŠŠ€e#›7oöÜÉŧŊķ{aũü38ž;w 3wî\æΝ[ŽcCCC]ĮĪ;—ņãĮpŲe—y˛‰Rx4‡"R.ĘĄˆYĘ ˆyĘĄØ‘ X6röėYĪž°E ëįɓpė˜gĪíoŊõ^^^ôíۗ'Nđúë¯ĶĒU+üüüfĐ AlÛļ €„„={ödôčŅĶŽ];V¯^ Ā¨QŖŠô5j#FŒ`Ĉåjcƒ >|¸kŲąc=öXÅߊ–<žC)3åPÄ,ePÄ<åPėH, ôė [ļĖüķĪž=wMŸ>{īŊ—!C†°dÉ‚‚‚ˆˆˆāā•W^aėØą,[ļŒģîēĢĐqk׎%,,ŒaÆąsįNüqîŊ÷^æΝËėŲŗ  OŸ>—lGVVéé餧§“‘‘Q꾛6mbéŌĨ´lŲ’aƕķ•KuįņŠH™)‡"f)ƒ"æ)‡bG*`ŲHŗfÍ<{ÂjZĀJIIaܸq <˜… Rŋ~}Ž=Ę /ŧĀŨwßÍķĪ?ONNëׯ/tlll,SĻLá‰'ž`÷îŨôęՋ[nš…Ī>ûŒÔÔT&MšÄ˜1c.Ų–ņãĮJhh(7ÜpCŠûūũīāá‡ÆËĢæDĪáp••åöūdĀ€ní;xđ`ētéB\\ũúõ#11Ņ­ã.\Xb/ļ3fĐšsgęÔŠÃ?˙ųĪ"ÛįĪŸOįΝéØą#ąąą$%%šuMwy<‡"RfĘĄˆYĘ ˆyĘĄØQÍųW´\ŌĻM›<{ÂjZĀōķķ#((ˆUĢVņĶO?púôiƎKzz:3fĖ`ɒ%øøøé: @ŨēuČÎÎĀétr×]w1oŪ<yäžyæˇÚ2a–.]ĘŌĨK]Ēâúč#Ν;įöå'NœČŦYŗhßž=ņņņ|øᇮųąJËĀ8p =zô(qŋ×^{ķįĪ3vėXWąšäå—_æŠ+Ž mÛļŽ í-ZD\\œkņõõeöėŲ$%%Ņ¸qc׹‡ƒI“&9 ((Čõ8##ÃížiŗfÍâæ›o.v[׎]éĐĄCąįúûß˙Î#Žēę*ēuëÆSO=ENNÉÅŌōđxE¤Ė”Cŗ”Aķ”CąŖ:Ļ ž“7Îc I8tļo÷ėšķė_+nœ…ןNąÖ÷›ŅVQ$&&§3ŋĢ¯žš3gθžĮÅÅņĮ?ūŅõŧā0Āūũû:öâs%$$”ØĂûšãâs5hЀcÕđ.Že‘7~˖-éÛˇ/+VŦp Ņ[ļl“&MâÛoŋĨaÆ=z´LĮŋ÷Ū{Ėž=›Į{ŒE‹UÚë¸páëÖ­cņâÅ8N~ķ›ßđÖ[o1vėX]Ãã9‘2SEĖREĖSŎÔËF8āų“ļooũLKƒb ’›ë Hņ ō×­{°Ôá„b†Ãá`ķæ͌=šO?ũÔ5,¯,Įtįw˛lŲ˛b `7ß|ŗk¨bqÛŨÕŧysnēé&üüü¨_ŋ>Æ ãĮ,÷ųŠS)9‘2QEĖREĖSŎTĀ’Ōå°vė¸äîeēc]Ú 8]ÚĢNīˇöûÅĶO?í*døûûĶ˛eK×ķ;vđüķĪĶŽ];ŧŧŧøüķĪ‹œqúôétčЁN:ŅĨK—"“ŧ×&99sįZ?KI7cÆ ’’’øîģīčÛˇ/)))ÜxãĖœ9“Ž;–z­âŽĪĖĖäāÁƒŽ}>ūøc5jDHHH‘ãį͛įŽX–ĄĄ9r$K–,!77—œœ–.]J׎]Ë}>ŠBh#Ũģw÷üI/.`õéãšsŸ9Tæũž{î9ž{î9ĀøčŖōë_˙ÚĩũškŽáˇŋũ-øÊœæĶO?åũ÷ßgÕĒU’ššęēamŗ`<đü™¨(xå(n+Ž¸â ŌŌŌxíĩ׈ŽŽæŲgŸ%--‡zČĩßsĪ=G—.]Ü:~˙ūũÜtĶMœ9s///BCCųüķĪ‹íUœ’ö›3gO<ņĮįĶO?å…^āŗĪ>Ŗ[ˇn >œĩk×ŌącGŧŊŊšōĘ+=>/YĨäPDĘD91K1O9;r8Ë:Á effHFFĻ›Sn›6mĸsįΞ=é’%0dˆõøąĮāÅKŨŨápđĖ3Ī°téRŌŌŌxöŲg1bgΜaÔ¨QlÚ´‰ēuëΒ˙žĀĖåđĘbkĐ`]o˜÷Ą8é5Ë ŧ‘kWĀ*m[ßž}™4i,Įa ĀÍ7ÃÅÉĪĢ͛W|Ģŧ'OžÄßßßcį|ņÅųųįŸyķÍ7=vNOŠ”ŠH™(‡"f)ƒ"æ)‡RU´öĸX6rúôiĪŸ466˙ņƍn’wĮšŸū™Ë.ģŒž}û˛víZŽ?ÎÖ­[ŦI̓Iøš1û4OA“`8}ŽĐ™ ~„öķČKŲēu+k׎å™gžáÜšsÜqĮÜ˙ũ9wM‘“cõŧ*ŽlítZEŦ„noī*ož[&NœČĮĖ|`ē)ÅĒ”ŠH™(‡"f)ƒ"æ)‡bGšËF<ŲÃÅĨiSˇ˙øcņ•‹wĮšŽ]ģ˛}ûvƎËūķk螗7˙;Ї;úB“`ĢûO}k_ēõ˜^žŠ¤dggŗgĪžũö[–,YÂ;īŧSŠwŧĢŽVŦ(ĶM¨¸ÆáÁ᧟Ŧå‰' &&ûšsÖŨ oŊÕÚwčPx÷]kčĄH5`‹ŠÔpĘĄˆYĘ ˆyĘĄØ‘ X6’a§áuté“'ÃĪ?Ã?Āũ÷Cxxū>įÎYsiũņÖ0Þ}áå—a×.sí–ZĪV9ŠĄ”Cŗ”Aķ”Cą#°lÄ×××t*‡Ã—_¯ŧ))đÕW0v,DFæīãtÂ÷ßÃcAÛļĐĻ Œo¸N2×vŠul›C‘D91K1O9;ŌXØgŦœœŧŊŊM7ŖęäæÂēuđÉ'Ö˛ukņûÕĢũúÁ!ÖŌŠ“u7D‘JPër(R )‡"f)ƒ"æ)‡Ri,qYģv­é&T-//čŲūö7Ø˛vė€)SāĒĢ NüũΟ‡¯ŋ†  kWkâÍ7Ã?˙ ›7[…0Šu9Š†”Cŗ”Aķ”CąŖ:—ŪE¤†hÛÖ*RM˜™™đÍ7đå—đŐœœŋ_z:ĖŸo-`MÕUÖŌ§5÷VŨēf^ƒˆˆˆˆˆˆˆĄ–4iŌÄtĒ€¸ņFkq:aįNXŧØ*j-_'5ŧ¸ åįņņĐģˇ5÷VīŪaâUH ¤Š˜§Š˜Ĩ Š˜§ŠŠ€e#õë×7Ũ„ęÉá€víŦå '6n„„kš¸ uæ ŦXa-yš7ˇŠYŨģCˇnÖŌ¸qUŋŠ”Cķ”Cŗ”Aķ”Cą#°ldĪž=4VQåŌŧŊķ‹P=d´~úÉē‹áĒUđðwoác’“­å?˙É_U¸ Õ­DG[3Šĩ”Cķ”Cŗ”Aķ”Cą#°DŧŊ­BT÷îpß}Öē#GŦbV^AkÍ8}ēđqXË…ųëüũ!6:v,üŗY3ļDDDDDDDĘÉát:ĻaZEoåX]deeáīīoēö” ÛļÁ† Ö˛~=$&Z“ÅģÃß:t°ŠYmÚ@ëÖųK``Ĩ6]Ē–r(bžr(b–2(bžr(ÕQEk/*`aŸÖΝ;iÛļ­éfÔššÖPÃõë­ĸÖƍ°ukŅᇗš_ĖjĶZĩ˛æÜjÖ "#­bRc(‡"æ)‡"f)ƒ"æ)‡RU´öĸ!„6rüøqĶM¨]ŧŧŦbSĢVpË-ųëO‚íÛaËĢ •÷sī^뎈KKŗ–~(ē­NkŽ­ŧ‚Vķæųķ \ jxbKJJĸE‹4oŪœ¤¤¤BÛ*3‡Ž_ūœŨũ‡ūũûŗ|ųr–-[F˙ūũųėŗĪxöŲgŲž};^^^tėؑɓ'Ķŋ˙Jkŗˆ ú}(b–2(bžr(v¤–Ô­[×t AčŅÃZ :}vî„={`÷nØĩËúš{7¤¤ŽėlHJ˛–ŌŽ™ŋ4iRøyŪ:u!Ž•™Ãšsį–ûØ3gÎpË-ˇpáÂūō—ŋpâÄ ^zé%FŽÉÁƒ=ØJķôûPÄ,ePÄ<åPėH,éŪŊģé&Hięׇ¸8kšØéĶų…­Ÿļîx¸o_ūŨKû”S§ŦbØŽ]Ĩ_ßߊ.ĄĄEŸ7n úĨįļˇŪz‹{īŊ—>}úđųįŸķúë¯3uęT<ˆ¯¯/ņņņŧúęĢtčЁ„„ @||<:uâ“O>!,,Œ÷Ū{^Ŋz1jÔ(fĪž]čüwŪy'ŗfÍbĈ >ŧĖmĖÉÉÁápP¯^=ČącĮxéĨ— ņČ{ Rč÷ĄˆYĘ ˆyĘĄØ‘ X6˛zõjzõęeēRõëCįÎÖRœ“' ´öíƒũûáĐ!8xĐZ.5Ą|V–ĩüüŗ{m )Zā -~iÜØîX MŸ>ņãĮ3dČæĪŸĪĻM›ˆˆˆāĀ×חäädĻL™Â]wŨÅ÷ßī:níÚĩ\}õÕ 6Œ3fđø㏓ĀŊ÷ŪËĩ×^ËųķįyėąĮHMMĨOŸ>—lGVVgĪžŦ˙q ŧčæūūū|ôŅGÜvÛmŽī‰ččh,XāÁwC¤zĐīCŗ”Aķ”CąŖÚų/N‘šĻaCčØŅZJ’•e´ ĩō–ŧõiipė˜{×zô(/ŧđ‡ríģ~ũúBĮÆÆÆ2eĘvíÚŌ3ØŊ{7Ŋzõ">>žáǓššĘ¤I“3fĖ%Û2~üxWĪ­ĢŽēŠ„„„BÛ/\¸ĀsĪ=Įšsįx÷ŨwÉĘĘ⡇âļÛncõęÕxyyUä­ŠT*`ŲHxx¸é&ˆIūūÖ] Û´)}ŋ  =RS­‚VjjáåâuYYî]?¯āĩc‡{û–\ā*ŽøåããŪyĢŸŸŪŪŪŦZĩŠŸ~ú‰øøx;v,^^^Ė˜1ƒ¨¨(†ęę•'44ȟŸ ;;°&hŋëŽģ˜7o<ōĪ<ķŒ[m™0aˇß~;ÁÁÁEļ'&&˛víZētéÂūđžūyÖŽ]ËÁƒ‰ŠŠ*ߛ R é÷ĄˆYĘ ˆyĘĄØ‘ X6RžÛPJ-TˇŽ5Š{“&îíæL~QĢāĪ’–“'Ũ;oF†ĩüŌķč’6Ė/f…‡[“Ķ7mš˙3īqppÅîʘ›i+āĖ!đkĄũĀËģØ]CBB˜;w.ƒ bđāÁ|õÕW„††âp8ČÎÎæØącŦ\š’sįΚ}ų‰'2kÖ,ÚˇoO||<~ø!-Z´¸dđØØXbccKÜŪĸE ęÕĢĮ–-[˜2e YYY¤ĻĻŌ¨Q#š¸ûYŠ!ôûPÄ,ePÄ<åPėH,Ųĩk—Æ9‹įųųAŗfÖâŽŗgK/p]\ËČpīŧ'OZËĨæđōķ+Z܊Š‚˜hŪÜúT|‘kûûđÜXø2ŽÁĀxz:´ŋ­ØËõîŨ›… ōĢ_ũŠAƒ1mÚ4ĻOŸÎĉ™:u*cƌĄQŖF=zÔ­—šråJĢ)Ûˇģ&mŋķÎ;+œíƍ3ū|&MšÄ_ūōŧŊŊšōĘ+yá…đö.ž@'RSé÷ĄˆYĘ ˆyĘĄØ‘Ãét:M7´ĖĖLÉČȨŅ•jMÔ'5Ōųķ֐ƒ \/îÎáUš€ĢUpiŧžžÉ@ÁoEĐøbN‰EŦ‚”Cķ”Cŗ”Aķ”CŠŽ*Z{Q û°233ktûEÜrá‚UČJIą&¨OI)ūņ‰îŸĶ\|AáâUÁíˇ4€ˇöAPHŠ§REĖSEĖREĖSĨ:ĒhíECm$55U_RbuëZC##KßīÔ)ë΋))°o$'CRRū˛oŸU hŦĸøâŋŦ˙ęôii­‰ō[ˇÎ_ōž+‡"Հr(b–2(bžr(v¤–=z”Ö­[›n†HõĐ A~Š899V+) ’æÂĶK?ßq °-ŨōøÃE÷ !""ēu+Zā ŠØäō"â6ũ>1K1O9;RËF4ŗHx{[“ģGEA›lžĨM¯ 4ī ûŌ­^]Å9v ˙cĮ`ëÖĸۂ‚ŦbVĢVÖRđq“&āåå%" ß‡"Ļ)ƒ"æ)‡bGš ûĖ%"唛ˇ‡Ā‡™%Ī5<æ/o8}ÚēâŽ]°{wáe˙~(ë×Ē¯o~1ëâWķæÖ°I‘L“¸{€] XkÖŦĄgĪžĻ›!R3mŽģŊÂw!\ģbņå´ š’“Ë^ÜōöļŠXÅˇZļ´†JŠH!ú}(b–2(bžr(Ց&q—ÜÜ\ĶMŠšÚßfŨ…đšąđeĻ5įU00$žžîVņ §^=ˆĩ–‹?oÍšĩ{7ėŲ“ŋėŪ {÷šsŜ0ĮęíõķĪ°tiŅí%MÔŧ[RKé÷ĄˆYĘ ˆyĘĄØ‘ X6jē "5[ûÛ`ÎpH[g_íg tSŠ9ŦWÚļĩ–‹åæZskå´..pefÎÇ­åģīŠn ,\Đ*XāŠŒÔŧ[b[ú}(b–2(bžr(v¤!„Øgá‰' 2Ũ ‘Z­RrčtÂŅŖ… Z \‡—ũœžžÖĂŭ–-ĄE ˆ‰??Īž‘*¤ß‡"f)ƒ"æ)‡RiĄ¸ėØąƒ^Ŋz™n†H­V)9t8 qck)îÜYYÖÃâ \ÉÉVīŽ‹=kŨ-ą¸;&‚ÕCĢE‹üĸVÁŸęŊ%՜~Š˜Ĩ Š˜§ŠŠ€%"RĶųûC—.Örąķį­"ÖÅC÷ėąŠ^gĪ΃­åûī‹nĢWĪęĨU\qĢeKkčĸˆˆˆˆˆˆi!öBxüøq‚ƒƒM7C¤VĢQ9ĖÍĩŠTy…­Ŋ{­%oŌø#GĘwŪāā’‹[͚Y0‘JTŖr(bCĘ ˆyĘĄTGB(.ú’1¯FåĐË ĸĸŦåĒĢŠn?uĘēkbÁĸVŪãŊ{­íÅ9~Ö­ŗ–’ŽY\qĢE םĨÂjTElH1O9;RËFŌŌŌhŲ˛ĨéfˆÔjļĘaƒĐąŖĩ\Ė鄴´ÂE­‚÷ī/~î­Ü\ØˇĪZ–//ēŨĪš7ˇ†(\ōÖŠĀ%n°UEj ePÄ<åPėH,qču"Æ՚:f-—_^tû… V‘Ē¤׹cş÷ĖØžŨZŠãë›_Ė*ŽĐŽ æĨöäP¤šREĖSŎ4ö™KD¤ÆČČ(<ąāĪää’'—ŋ”zõ ļ..r5iĸ—ˆˆˆˆˆ­Ŋ¨€…} XëÖ­ŖGĻ›!RĢ)‡ātBjĒUČJJĘ_ >?}ē|įŽ[ךHž¤!ŠMš@uNŽé”Cŗ”Aķ”CŠŽ4‰ģ¸dgg›n‚H­§z€Ãa  ‡Ë.+ēŨé„ôôĸŽŧ"WiĖ_¸×Åâx{Cd¤Uä*¸DGį? Ō<\՜r(b–2(bžr(v¤–„„„˜n‚H­§V‡BC­%>žčv§Ķšcëâ^[ \'Oîœkúũûáûī‹ßĮßŋpAëâ"WTøøxæĩJš(‡"f)ƒ"æ)‡bG*`ŲHxx¸é&ˆÔzĘa5āp@ŖFÖR\×y§Nœ(žĀ•WŧJK+ųüYY°m›ĩ”$"ĸô"WX˜zqU"åPÄ,ePÄ<åPėH,ŲļmŊzõ2Ũ ‘ZM9Ŧļ–nŨŠßįôi8pĀē“âž}VQ+īqŪRÚDķ‡[˚5Åo÷ņąŠYŅŅV­ĻM­Ÿ‡…iÂųrREĖREĖSŎTĀšXũúĐļ­ĩĮé„ŖG ´..r:díWœsį`÷nk)I:Ö|\ļ >nŌÄēķĸˆˆˆˆˆÍŠ€e#­[ˇ6Ũ‘ZO9Ŧ%hÜØZēw/~ŸķįáāÁĸ=ˇō ]ÉÉ%Ī՝Lií/ž¸•÷¸iShĐ b¯ˇ†QEĖREĖSŎTĀ˛‘ŦŦ,5jdē"ĩšr(.õęALŒĩ”$3RRŦáŠyKÁį))ÖKâtæW\ˇŽäũ‚ƒ­BV“&V¯Žâ~6ižžå}ĩՊŨs˜””D‹-hŪŧ9IIIUv]Į/ķļ9KęYx‘ūũûŗ|ųr–-[F˙ūũŲļm?ü0?üđ‡ƒn¸W_}ĩ\ˇŅ–ęÍîŠ ”Cą#°läđáÃ4oŪÜt3Dj5åPĘ$ ĀZ:t(yŸ3gŦž\Ĩē†ÜܒĪqü¸ĩlŪ\z{BBJ/råũŦæ….å°rĖ;ˇÜĮž;wŽëŽģŽđ /°gĪŪ|ķMœN'ŗgĪö`+Ĩ:PEĖSŎTĀŠÎüü U+k)IvļUÄ*ŽWÁĮįĪ—~­cĮŦe˖Ō÷ .ŊĀn- ęn‹Uā­ˇŪâŪ{īĨOŸ>|ūųįŧ˙ūûL:•ƒâëëK||<¯žú*:t !!O§Nøä“O ãŊ÷ŪŖW¯^Œ5ĒHAéÎ;īdÖŦYŒ1€áǗš[ˇn%99™Ž]ģōčŖröėYŪ|ķMæĖ™ÃĢ¯žJ`` GŪ ą/°lä˛Ë.3Ũ‘ZO9#ęÔɟ÷Ē$N§Õ ëāAk‚ųC‡ō_üŗ´;,B~Ž­[KßĪ×7ŋ˜…Ÿ\=VėĒM9œ>}:ãĮgȐ!ĖŸ?ŸúõëÁ<€¯¯/ÉÉÉL™2…ģîē‹īŋ˙ŪuÜÚĩkšúęĢ6l3fĖāņĮ'!!{īŊ—k¯Ŋ–ķįĪķØc‘ššJŸ>}.ŲŽŦŦ,ÎūōšŠ[ˇn‘‚Txx8^^^ėŨģ—Ÿ~ú‰;w››Ëž={č^Ō\rR#ÕĻ ŠTWĘĄØ‘ X6’˜˜Hˇ’n /"UB9”jËᰆ†„@§N%īįt‰… Z%ģΜ)ũšgĪZ“Õ''_ē}>>%ˇ..|•ZėĒ-9LIIaܸq 2„… Rˇn]Ž=Ę /ŧĀĄC‡\ûŽ_ŋžĐąąąąL™2…]ģv1cÆ v˙rGĖ^ŊzĪđáÃIMMeŌ¤IŒ3æ’m?~ŧĢįÖUW]EBBBĄí‘‘‘<˙üķ<ųä“ÄÅÅQ§NęׯĪéĶ§ŨžSKjŽÚ’A‘ęL9;RËFÎ_jhˆˆT:åPj<‡Ã" ąą%īįtZ“Đ_\Ø:tŽ)ŧ¤§[û—æÜšKßu1OŨēÖ CCķ—Īũ‡ŒŒüm!!V/5›ņķķÃÛۛUĢVņĶO?ĪéĶ§;v,^^^Ė˜1ƒ¨¨(†ęę•'44ĀUôĘÎÎŦ Úīēë.æ͛Į#<Â3Ī<ãV[&L˜Āíˇß@pppąû<ūøãŒ5Š={öæę[ÚįLj$ũ.1O9;˛ßßæją   ĶMŠõ”CŠ5kØ_``é“Đƒ5GWZZŅÂÖáÃE×ĨĨ]ēØuáB~ąŦmŠkkpp‘BW‘įûųšũVxTn¤­€3‡Ā¯ „ö/ībw aîÜš 4ˆÁƒķÕW_ŅŽ];ŲŲŲ;vŒ•+WrîÜ9ˇ/?qâDf͚Eûö퉏įÃ?¤E‹ôęÕĢÔãbcc/Yˆzųå—ņķķÃĮĮ‡ &püøqžzę)üLŊ×RiôģPÄ<åPė¨Ú°&OžĖ‚ Øž};~~~ôéĶ‡)SĻĐŽ]ģR[ž|9?ü0[ļl!22’ &pĪ=÷TQĢĢ‡¨Ōæ>‘*ĄŠŖNkr÷&M.ŊovļÕcĢ´BWjĒUčJKŗŠY—âtæOPŋc‡{möõÍrŲ¨Qņ‹{^ŋž{į/Îö÷ášąđe&‚!đôth[ą‡ôîŨ›… ōĢ_ũŠAƒņõ×_3}út&NœČÔŠS3f 5âčŅŖn5aåʕVSļowMÚ~įw^˛€åŽôôtŪyį233iŪŧ9/ŋü2?üp…Ī+Տ~Š˜§Š9œÕlâk¯Ŋ–áÇĶŗgO˛ŗŗ™8q"›6mbëÖ­4hĐ ØcöîŨK§Nøã˙ČŨwßÍ÷ßĪØąc™;w.7ŨtĶ%¯™™™I`` xú%U™ÕĢW{ä/˜"R~ĘĄHr:áäÉübVz:¤ĨąoŨ:šųųšžÜÆɓ•ÛĻ’ _yĪķ†gYKŪãË`蝐 ü›™h|1§Ä"–HuŖ߅"æ)‡RU´öRíz`-^ŧ¸Đķ™3gÆēuë¸ōĘ+‹=æÍ7ߤYŗfL›6 €:°víZ^~ųeˇ X"""R9`-­ZšVŠĨYIi?{ŽÍ/l,nå=NOĪīąuôčĨīĘxņų´ˇ_pE‹Wüō<Ģg֜á%'ąģjWĀēXFF`ÍķP’~øÁƒZ7dČūõ¯qáÂ×ĨvײeKĶMŠõ”CķJÍĄ¯/4mj-î:sĻpA+īņĨž_ę.yÚĢ(ZŧĘãÄV˜ļÂûģßnCôģPÄ<åPė¨Z°œN'?ü0}ûöĨS)ˇü>|ø0ááá…Ö…‡‡“Mzz:M.šsãÜšs…&4ÍĖĖôlà šøC"Rõ”Cķ<žC?ŋ˛Ŋ páĢārâ?ž˙3p+ŧ™XúšŽcMė.RčwĄˆyĘĄØQĩ.`?ž7ōŨwß]r_‡ÃQčyŪÔ^¯kĸøgŸ}ļČúĩk×Ō AēwīÎļmÛ8sæ 6¤E‹lܸ€æ͛“››Ëūũûˆ‹‹c÷îŨdeeŅ AÚļmˆ kâŗ999Ž÷ģ[ˇnėÜš“S§NáīīOëÖ­ILL :://¯BŸŲŊ{÷rōäIüüüčĐĄƒëũnÚ´)õęÕcīŪŊŽ÷{˙ūũœ8qētéš5k\ŸŲ ¸ŪīØØX>ĖącĮŠŧßaaaēŪīöíۓžžNzzēë3›÷~7nܘƍŗ}ûv×g6##ƒÔÔÔ"ŸŲ"""ØēuĢë3{ęÔ)×ûŨŗgO6nÜČšsį "::Úõ™mŅĸįĪŸ'%%Åõ™-é;âėŲŗx{{WËīˆcĮŽQ§N}Gč;Âöß[ˇnåāÁƒÕį;ĸukëũŽŒ¤yīŪEūq0ņZüį÷pŒ’Ã9¯~ŪļMßúŽ¨öß[ļláāÁƒÕö;ĸ:˙=ĸ:˙[Cß5ë;âÔŠSVËīˆęū÷}GTŪwÄŝ‹ĘĒÚMâžįžûîã“O>áÛoŋĨE‹Ĩî{å•WŌ­[7^yå×ē?ū˜[oŊ•Ķ§OBX\ŦččhMâ."ĻŠ˜Wãr˜›ˇ‡Ā‡™Å#tÃ`Î1́%5BË ˆ )‡RUtw¯JhS…8NƏĪ‚ øæ›o.YŧëŌK—.-´nɒ%ÄĮĮ;˙•…;čŅŖ‡é&ˆÔzĘĄˆy5.‡^ŪđôtënƒwĪģ áĶĶUŧ’ŖÆePƔCąŖjWĀ7nsæĖáƒ> aÆ>|˜Ã‡sĻĀD¨úĶŸ¸ãŽ;\Īīšį’““yøá‡Ųļm3fĖā_˙ú>ú¨‰—`L^[1G91¯Fæ°ũmđÅĢ§UVá*ëųsŦí"5DĖ ˆÍ(‡bGÕnŦ7Ūx€ūũûZ?sæLFĀĄC‡\ã”Á3ēhŅ"zč!^ũu"##yõÕWšéĻ›ĒĒŲՂ&ę1O91¯Ææ°ũm0g¸uˇÁ3‡Ā¯ „öSĪ+ŠqjlElD9;Ēv,wĻäš5kV‘uW]u•k¡ÚĘ.C!Ej2åPÄŧC/oīoē"RŖ3(bĘĄØQĩB(åcē "ĩžr(bžr(b–2(bžr(v¤–äŨZSDĖQEĖSEĖREĖSŎTĀ‘jM,iŪŧšé&ˆÔzĘĄˆyĘĄˆYĘ ˆyĘĄØ‘ X6’““cē "ĩžr(bžr(b–2(bžr(v¤–8pĀtDj=åPÄ<åPÄ,ePÄ<åPėH,ŠÖN§ĶiēĻeffHFFĻ›SnįĪŸ§^ŊzĻ›!RĢ)‡"æ)‡"f)ƒ"æ)‡RU´öĸX6˛sįNĶMŠõ”Cķ”Cŗ”Aķ”Cą#°läÔŠSĻ› Rë)‡"æ)‡"f)ƒ"æ)‡bG*`Ųˆŋŋŋé&ˆÔzĘĄˆyĘĄˆYĘ ˆyĘĄØ‘ X6ŌēukĶMŠõ”Cķ”Cŗ”Aķ”Cą#°l$11ŅtDj=åPÄ<åPÄ,ePÄ<åPėH,ŠÖTĀ˛‘ččhĶMŠõ”Cķ”Cŗ”Aķ”Cą#°lÄËKœ"Ļ)‡"æ)‡"f)ƒ"æ)‡bGúTÛHrr˛é&ˆÔzĘĄˆyĘĄˆYĘ ˆyĘĄØ‘ X""""""""R­9œN§Ķt#LËĖĖ$00ŒŒ L7§ÜΜ9ƒŸŸŸéfˆÔjĘĄˆyĘĄˆYĘ ˆyĘĄTG­Ŋ¨–ėŨģ×tDj=åPÄ<åPÄ,ePÄ<åPėH,9yō¤é&ˆÔzĘĄˆyĘĄˆYĘ ˆyĘĄØ‘ X6ĸ.ĸ"æ)‡"æ)‡"f)ƒ"æ)‡bGš ûĖuáÂęÖ­kē"ĩšr(bžr(b–2(bžr(ՑæĀ—õëכn‚H­§Š˜§Š˜Ĩ Š˜§ŠÕ1Ũ€ę ¯ZffĻá–TĖŠS§jükŠé”Cķ”Cŗ”Aķ”CŠŽō>“å¨ųÜEGGn‰ˆˆˆˆˆˆˆˆ}>>Ļ›"Rk)‡"æ)‡"f)ƒ"æ)‡bWšÄ]DDDDDDDDĒ5õĀ‘jM,ŠÖTĀ‘jM,˜>}:-Z´Ā×ח=z°bÅ ĶMą­oŋũ–ĄC‡‰Ãáā“O>)´Ũét2iŌ$"##ņķķŖ˙ūlŲ˛ÅLcEljōäÉôėŲ“† ƍ7ŪȎ; íŖ,ŠTŽ7Ūxƒ.]ē@@@Ŋ{÷æ‹/žpmWEĒÖäɓq8<øāƒŽuĘĄØ X5Üūķ|đA&NœČ† čׯ×]wûöí3Ũ4[:uę]ģvåŸ˙ügąÛ_|ņEĻNĘ?˙ųOÖŦYCDDƒ âäɓUÜRûZž|9ãƍcÕĒU,]ē”ėėlĖŠS§\û(‹"•+**Š^xĩkײvíZŽžújn¸á×?Ž•A‘ĒŗfÍŪ~ûmētéRhŊr(vŖģÖpŊzõĸ{÷îŧņÆŽu:tāÆodōäÉ[&b‡ƒ?ū˜oŧ°ū—+22’|Į€sįÎΔ)S¸ûîģ ļVÄžŌŌŌ cųōå\yå•Ęĸˆ!!!!ŧôŌKüū÷ŋWEĒHVVŨģwgúôéüõ¯%..ŽiĶĻéwĄØ’z`Õ`įĪŸgŨēu <¸ĐúÁƒŗråJC­ŠŊöîŨËáÇ eŌĮĮ‡ĢŽēJ™ŠD€õgPEĒZNN~ø!§NĸwīŪĘ H7nŋúÕ¯8p`ĄõĘĄØQĶ ōKOO'''‡đđđBëÃÃÃ9|ø°ĄV‰Ô^yš+.“ÉÉÉ&š$b{N§“‡~˜ž}ûŌŠS'@YŠ*›6mĸwīŪœ={>ūøcbcc]˙8VE*ׇ~ČúõëYŗfM‘mú](v¤– 8ŽBĪNg‘u"Ru”I‘Ē3~üx6nÜČwß}Wd›˛(RšÚĩkGbb"'Nœ`ūüųÜyį,_žÜĩ]Š<û÷īį`ɒ%øúú–¸Ÿr(vĸ!„5Xãƍņöö.ŌÛ*55ĩHĨ]D*_DD€2)REîģī>.\ȲeˈŠŠr­WEĒFŊzõhŨē5ņņņLž<™Ž]ģōĘ+¯(ƒ"U`Ũēu¤ĻĻŌŖGęÔŠC:uXž|9¯žú*uęÔqeM9;QĢĢW¯=zô`éŌĨ…Ö/]ē”>}új•HíÕĸE """ eōüųķ,_ž\™ņ §ĶÉøņãY°`ß|ķ -Z´(´]Y1ÃétrîÜ9eP¤ \sÍ5lÚ´‰ÄÄD×ĪmˇŨFbb"-[ļTÅv4„°†{øá‡ųŨī~G||<Ŋ{÷æíˇßfßž}ÜsĪ=Ļ›&bKYYYėŪŊÛõ|īŪŊ$&&BŗfÍxđÁyūųįiĶĻ mÚ´áųįŸ§~ũúŒ9Ō`ĢEėeܸq|đÁ|úé§4lØĐõŋˁøųųáp8”E‘Jöį?˙™ëŽģŽččhNž<ɇ~HBB‹/VEĒ@Æ ]s?æiĐ 5r­WÅnTĀĒá~ûÛßrôčQž{î9:D§NX´h͛77Ũ4[Zģv- p=øᇸķÎ;™5k&LāĖ™3Œ;–ãĮĶĢW/–,YBÆ M5YÄvŪxã ú÷ī_hũĖ™35j€˛(RɎ9Âī~÷;:D`` ]ētaņâÅ 4PEĒåPėÆát:Ļ!""""""""Ŕ%""""""""՚ X""""""""R­Š€%""""""""՚ X""""""""R­Š€%""""""""՚ X""""""""R­Š€%""""""""՚ X""""""""R­Š€%"""ÄÄÄ0mÚ4ŖmHJJÂáp˜˜h´Rûčķ;vė ""‚“'OVúĩRSS %%%ĨŌ¯%""öĄ–ˆˆÔJ‡æžûîŖe˖øøøÍĐĄCųúë¯M7͈5kÖ0fĖ˜J;˙¨QŖp8Ĩ.ŅŅŅ:tˆN:UZ;*ËäɓéŲŗ' 6$,,Œoŧ‘;vxü:ũû÷wŊ_>>>4mڔĄC‡˛`Á_Ģ6Ņį&NœČ¸qãhØ°aĨ_+,,Œßũîw<ķĖ3•~-ą‡Ķétšn„ˆˆHUJJJâŠ+Ž ((ˆgŸ}–.]ēpážüōKŪ~ûmļoßnē‰E\¸pēuëšnFšeddpæĖ×ķ&Mš0sæLŽŊöZ×ēˆˆM+VLL ŗfÍĸ˙ūnííĩ×2|øpzöėIvv6'NdĶĻMlŨē• xŦ]ũû÷§mÛļ<÷Üs\¸p””>ūøcūņ0jÔ(Ū~ûm]Ģ:Ņįŋr8p€–-[ōķĪ?U%×Ü´i—]v$88¸JŽ)""5›z`‰ˆH­3vėX?ūø#7ß|3mÛļĨcĮŽ<üđÃŦZĩĘĩßž}û¸á†đ÷÷' €[oŊ•#GŽ¸ļOš4‰¸¸8fĖ˜AŗfÍđ÷÷įŪ{ī%''‡_|‘ˆˆÂÂÂøÛßūVčú‡ƒ7ŪxƒëŽģ???Z´hÁ˙û_×öŧĄD}ôũû÷Į×ח9sæ0sæL:tč€¯¯/íÛˇgúôéŽãΟ?ĪøņãiŌ¤ žžžÄÄÄ0yōäBímÖŦ>>>DFFr˙ũ÷ģļ]<„ĘŨ×ūī˙›˜˜>|x‰Ã‰ˆˆp-AAA…Ö]<„*!!‡ÃÁ—_~IˇnŨđķķãęĢ¯&55•/žø‚:Āˆ#8}ú´ëZN§“_|‘–-[âįįG׎]™7o^ÉXŧx1ŖFĸcĮŽtíڕ™3g˛oß>Ö­[WėūN§“ríĩג÷‰'Nœ YŗfLœ8ąÔkÕ¯_ŸˆˆĸŖŖšüō˙2e oŊõīŧķ_}õ•kŋ””~ûÛßLŖF¸á†HJJ*tŽ3fĐącG|||hŌ¤ ãĮwmĶįŋö|ū?úč#ēvíZ¨x•œœĖĐĄC ĻAƒtėؑE‹šļoŨē•ë¯ŋÂÃÃųŨī~Gzzēk{nn.SĻLĄuëÖøøøĐŦYŗBŸ…Ν;ÁĮ\jÛDDD\œ"""ĩČŅŖG‡ÃųüķĪ—ē_nnŽŗ[ˇnΞ}û:׎]ë\ĩj•ŗ{÷îÎĢŽēĘĩĪ3Ī<ãô÷÷wŪ|ķÍÎ-[ļ8.\čŦW¯žsȐ!ÎûîģĪš}ûvįŒ3œ€ķ‡~p85jä|įwœ;vėp>ųä“NoooįÖ­[N§Ķšwī^'āŒ‰‰qΟ?ßųķĪ?;SRRœoŋũļŗI“&ŽuķįĪw†„„8g͚åt:Î—^zÉíüöÛoIIIÎ+V8?øā§Ķétū÷ŋ˙u8-ZäLNNvŽ^ŊÚųöÛoģÚÔŧysį?ūņ2ŋöaÆ97mÚäüöÛoÎ?˙ųĪnũ9Ώ?ū¸Đēŧ×ŊaçĶét.[ļĖ 8/ŋürįw˙ßŪũĮD]˙qp‚"x†Ä!Ĩ$”PāEe–Ĩ ũƒ&T(‚B™S ]Ĝ“š\Ö(É:#F&*Ĩ.ŦÁÁYƒr6/1EĮëû_>_Î$ ŋ—<ۍģ÷Ŋ>Ÿ÷įũæõĪįĩ÷į}ß/ÍÍÍ(ąąą˛téRinn–††™9sĻŧ÷Ū{ĘyŪ~ûm™7ož|ûíˇb4åŖ>GGG9yō䄮mx>Nœ81áøĩˇˇ 9wîܘ1ŋũö›xxxHqqąˆˆ$''Kdd¤üõ×_c+7n´j7›Íâáá!ëÖ­‘ĢW¯JPPŦ]ģVôzŊ´ļļĘęÕĢ%$$DúûûED¤¤¤Dœœœ¤¸¸XÚÚÚ¤ąąņ–r€ų˙īĪ˙å˗KFF†EÛŗĪ>+‰‰‰ĸ×ëÅh4JUU•œ:uJDD.\¸ žžž˛uëV1 ŌÜÜ,‰‰‰¯Ÿ——'RVV&ōŨw߉VĢĩčcÕĒUōĘ+¯LhΈˆˆXĀ""ĸ)å‡~ĸĶéƍĢŠŠéėėTÚZZZ€466ŠČĐMŦJĨ’ŪŪ^%FŖŅˆŋŋŋ˜ÍfĨ-$$D •ĪŦnŖĸĸ”âÃđėpac˜Z­VnȇíÚĩK–,Y"""YYYōä“OĘāā ÕxöîŨ+ÁÁÁcGFŪĀßęØsss%**jÔķßčīÜĀ×ÖÖ*1………@ŒFŖŌöú믋FŖ‘?˙üSœœœäĖ™3įNKK“””” ]›Čí°eŲ˛esĶØĪ?˙\eëÖ­ĸRŠ¤­­mÜøą X"C9”””$""‹\čīīggg9vė˜ˆˆøúúĘļmÛF=ķjå˙Ã?,m ,üüüQãˇoß.K—.ĩhëęęŌÖÖ&ŊŊŊâččhU°ēŅæ͛%..nÜ""ĸa|„ˆˆĻųīãZvvvãÆ ¨Õj¨ÕjĨ-,, îîî0 J›ŋŋŋÅĻĮŪŪŪ ƒŊŊŊE[wwˇÅų—,Ybõyäy 22RyßĶĶƒŽŽ.¤ĨĨÁÅÅEyíŪŊFŖĀĐFŅ?ũôBBBššåø•+Wĸ¯¯xõÕWqäČ üŖc÷ņņąį?!<<\yīíí •J…€€‹ļá~[[[qũúu$&&ZĖĶĮŦĖĶh222,â;;;‘””dÕ66l€^¯ĮgŸ}vĶؕ+WbŊ(,,ÄŪŊ{<Ą>F#"J^755ĄŖŖŽŽŽĘõĪ˜1ׯ_‡ŅhDww7.\¸€§žzjÔs1˙§Vū÷õõÁÉÉÉĸ-;;ģwīFtt4vîÜ Ŋ^¯|×ÔԄ'NXô1oŪ<€Ņh„Á`@˙˜ų5ĖŲŲŲâņG""ĸņLû_Ņ;;; ŧđ cƍ,Œ×~ãÆŌvvvŖļ ŪôÚnėoäæßÃĮkĩZDEEYÄ988"""đË/ŋāčŅŖ¨­­ÅĒUĢ€/žøjĩmmm8~ü8jkką~ũzėŲŗ§N˛ēŪÛûDÆųwėįfķ;üˇēē~~~qŽŽŽcöQPP€-[ļ(ŸãââPTTd1מžž7ŊÖŦŦ,|ũõ×hhh˜ĐfØ׎]CSSĐŪŪ~Ķøą˜Íf´ˇˇcŅĸE†æáŅGEyyšUėŦYŗ, LŖaūO­ü÷ôô„Éd˛hKOO‡FŖAuu5jjj”"kVVąlŲ2YËĮĮįĪŸŸĐØ.]ē„YŗfM(–ˆˆˆ+°ˆˆhJ™1c4 Ū˙}\ŊzÕęû?ūøĀЊ‹ÎÎNtuu)ßĩļļâōåË ŊíëšYüđįá Ŗņöö†ŸŸΟ?ĀĀ@‹×Üšs•8777$''CĢÕĸĸĸ_~ų%.]ē`hĩÃķĪ?ũû÷ãäɓ8{ö,Ν;gÕ×d}2………ÁŅŅVķ4rE͍ŧŧŧ,b§M›???Ģļąˆ6l؝N‡úúz‹˙Éxrrr`ooŖGb˙ūũ¨¯¯˙Ûc€ƒÂd2áĨ—^0TĖiooˇW`` ĻOŸWWWøûûŖŽŽnÔķ1˙§Vū/\¸­­­VíjĩĐétČÉɁVĢ0”_---đ÷÷ˇęįžûîCPPœĮĖ¯a?˙ü3.\x{ƒ&"ĸ)ƒ+°ˆˆhĘ)))Ác=†Å‹Ŗ  áááĀņãĮQZZ ƒÁ€„„„‡‡c͚5(..ÆĀĀÖ¯_ØØX‹G›nÕálj˜˜”——Ŗąą÷˜üü|dggÃÍÍ IIIčīīĮ?ū“É„7Ūxûö탏yäØÛÛãđáø˙ūûáî˛2˜ÍfDEEAĨRá“O>ŗŗ3xāĢ~&{ė“ÉÕÕ[ļlÁæ͛188ˆ˜˜ôööâĖ™3pqqAjję¤ô›™™‰O?ũ•••puuÅŋ ũúœŗŗķ¨ĮTWWãÃ?ÄŲŗgˇŪz ŠŠŠĐëõđđđŗ¯k׎áâŋĀīŋ˙N‡}ûöaŨēuˆŦYŗ{öėÁōåËQPP€ŲŗgŖŗŗ:ššš˜={6ōķķ‘‘‘///$%%áʕ+8}ú4˛˛˛˜˙S,˙5 ŌĶĶa6›•m›6mBRR‚ƒƒa2™P__¯đ233ĄÕj‘’’‚ÜÜ\xzzĸŖŖ‡‚VĢ…““Ū|ķMäååáŪ{īEtt4zzzĐŌŌ‚´´4˙[}øîģīŪ™É!"ĸ=ŽĀ""ĸ)gîÜšhnnF||]yiii’}ŨŨŨņÜsĪaĮŽ „‰‰ æΝ ¸qã^{í5¸¸¸ĀČČ˜;w.Š‹‹%įXąbÚˇo XZZÂĮĮ3gÎ_ĪÍÍÅ´iĶÄ)–vvv Áæ͛5ú\Qmī׉'ĐĩkW˜™™ÁĶĶŸ}öJJJ*˙Ãģ'00]ģvÕhWĢÕhŪŧ9 Pã÷Y™[ˇn@•ŖĒ*|îŸBX6 ¯˛wwwÉy~üņGtîÜæææ°°°@Ÿ>}pęÔŠö­ļjķ^*›BX•šôũʕ+xųå—áėė ccc4kÖ ={öDtt´¸OeSįΝ‹N:ÁÎÎVVV ÂÚĩkĢ-J5 ß|ķ Hî{ŲŸÍ7ß|ƒ'Ÿ|077GÛļmąpáBIÎSöŲbccąnŨ:ÉšRRR0|øpŧûîģøä“O´fÄQ DDD¤X7nÜ/ŋürö‰‰'N”´GDD„™3gŠmŨēu‘‘‘bÛ­[ˇ}}}ÁÔÔTHNNÛŖŖŖÂW_}%ļ}øá‡á­ˇŪ’\kĶĻMaãƍb›››› ¯¯/ü÷ß’}_{í5ÁÂÂB¸zõĒ¤ũķĪ?įĪŸA&Mš$ØØØ<đŊˇiĶFxá…¸OYŸË<ĖũŠˆˆėëįį'ôéĶį×MOOŒŒŒ$įA‡Bŋ~ũāėė •J…]ģvÕęø˛Š ÷˙˜››7L‡‰ˆˆ*8pā€ŌŠIuėØžžžØŋŋ¤ŨÉÉ ÁÁÁâļ iåëë ¸zõĒÆ5‡ &Ų 1{ölÜēu ŠŠŠĩęoE§NÂķĪ?&Mšˆį1bÔj5âââ$ûÚÚÚ⊧žzčk) X÷ÉÉÉAûöíņõ×_?ÔņĶĻMCJJŠäĮĪĪ/žøb=÷”ˆˆM›6…™™âããk´˙ƒÖrvv֘Nggg§ąŸ‘‘‘Fģ‘‘ ??_cGGGÉļš4iĸq­ĘútķæMüōË/044”ü”M]++āŧōĘ+XˇnŽ^ŊŠÂÁÁ:uŸū)žëĢ¯žÂ{īŊ‡]ģvĄG°ŗŗà /ŧ€‹/j\ˇLmīWeĶōŒkT,3f ’““Å>oŪŧ’âYMŪįƒ4kÖ ŖGÆʕ+qæĖzöė‰ß˙IIIpqqyāūe–””}¯_ŋ.YĪŠžÜ¸q͛7ˇ‹‹‹qëÖ-bĪũ‹§ĨēvíÚaūüų•žģâ(°ŅŖGcôčŅČÉÉÁĄC‡đá‡âšįžC\\ÜÜÜ`nnŽšsįbîÜš¸yķĻ8Ģ_ŋ~ˆ­ôüyŋúôéggg„……ĄOŸ> C§N4֎Ēî}ÖƓO>‰ĐĐPėÚĩ ŠŠŠppp¨rßņãĮc˙ūũØŗgÚˇo/y­ė>lÛļ­Ö}€'NHļ=<›DDDēD%Õ<æĻRŠ°sįNŧđ €üÆ Ãųķį%ÃāŌĩ¯î_ÄļgĪž°˛˛ÂΝ;ĢËDDDfΜ9˜;w.ŌŌŌdm-""""ĸ2œBX PĢÕHMM­vMĢøøx8pģwīn¤Ūé&°î“K—.‰ÛņņņˆŽŽ†ŧŊŊ1lØ0Œ1‹/F`` ŌĶĶņ÷ßŖmÛļčÛˇ¯xÜēuëāääôĀ'Qõ8…đ>áááčŅŖ‡Fûȑ#ą~ũzáã?Æwß}‡ääd4iŌ;wÆÜšsÅE8KJJāææ†#FTųXp""""""""Ē°ˆˆˆˆˆˆˆˆHŅôäîŅƒ°€EDDDDDDDDŠÆEÜQēfÕõë×aii •J%wwˆˆˆˆˆˆˆˆtŠ ¸{÷.œĄ§WûņT,`¸~ũ:\]]åî‘NKLL„‹‹K­c €ĨĨ%€Ō›hee%soj¯¸¸üŖ$Ræ‘H9˜G"å`‰”Y$9eeeÁÕÕUŦÁÔ?š€8mĐĘĘJ+ XčÔŠ“ÜŨ "0DJÂ<)ķH¤ Ė")ÁÃ.ŨÄE܉ˆˆˆˆˆˆˆHŅXĀŌÎÎÎrwˆîa‰”ƒy$Ræ‘H˜EŌf,`éšģ@D÷0DĘÁ<)ķH¤ Ė"i3ŽĨŽ\š{{{šģAD`‰”„y$Ræ‘**))AaaĄÜŨx$]šråĄĐ&ĒŽĄĄ!ôõõėü,`QŖ(,,D||ĸúÆ–¸té’Ü] ĸ{˜G"å`‰”ƒy¤z‘¸8<ČM’ļį&—ļ7pkúôéøčŖđë¯ŋbāĀb{aa!>úč#œ>}ģvíB|| [[[āå—_Žtøų_ũ%^kūüų8zô(Ž_ŋ___¸ģģãØącxõÕWņíˇß>T?ˆjŖē<Qãa‰”ƒy$R}}ûH{)n áŅŖGŅŋ<ûėŗJįûnŪŧ‘‘‘UŗråJ´hŅK—.øúú"22Ÿū9ØŨ–U\\\•SåvėŪ|HJ*osqžü¨ęéŦwīŪÅŗĪ>‹šsį⊧žÛ§OŸŽéĶ§‹Ûîîîøõ×_ÅWEÛļm{¸7SÁĐĄCņŪ{īĄ  ÆÆÆØģw/Úˇo¯ą_¯^ŊÄ)-ĶŊ{wL›6 Ī=÷\ûDT呈ķH¤Ė#‘2äįįÃÂÂBîn=ōĀzâ‰'°˙~ÄÅÅNŸ>ūų}ûö­ō˜ŖG"44TŌÖ§ODFFĸ¨¨¨AûĢd;vƒI‹W@’’T8Đ;vTuÜÄÆÆbŅĸEâČĻēNÉTŠTâBđ…‡‡ÃÅÅK–,ÁĒUĢāââ‚ŨģwŖ{÷îxüņĮáįį‡€€¸¸¸āģīžÃ7ß|•J…õë×׊?DDDDDDD¤=7ëŊ÷ŪCff&||| ¯¯ĩZųķįcȐ!UsãÆ 4kÖLŌÖŦY3#==NNN’× PPP ngeeÕī›hd­[ˇÖhSĢKG^=hžŠSūũ˛Q¤e ú9#GŽŦöē 5îãæ͛/ŋü˛¤Ŋ{÷îH’VØK–, 2ĢV­ÂōåËqūüyDDDāäɓ5ž.PZ$#j,•å‘ˆäÁ<)ķH¤ &&&rwčĄ)Ž€õã?bãƍøá‡āīīččhL:ÎÎÎ,Ē¨T*ÉvY1æūvøôĶO+  sss!&&yyy°´´„‡‡Μ9 t]Ļ’’$&&(}bßĨK— sssx{{ãÔŠSčëëãęÕĢ€víÚ!!!YYY011ŋŋŋXŒqvv†‰‰ Ž\šhĶĻ ’’’pįÎ! @\˙ÉŅŅ¸téĀŌŌiiiČČȀ‚ƒƒązu ’’|x¯wŪY…ĨK_GÛļm†ā‹/ž@zz:LLLĐĻMLš4 ČËËC=āëë ___üũ÷ß°˛˛ÂėŲŗŅŋŧūúëSûöí‹Å‹‹ȲõŗŠ‹‹ÅÂUPPbcc‘›› xyy‰EÅ[ˇn!%%ÁÁÁFFFNž<‰Ë—/ãėŲŗđņņATT”xŋ ÄÂZÛļmqíÚ5dffŠīĨl*Ē““ĖĖĖpųōe€ŋŋ?Ž_ŋŽÛˇoÃĐĐAAAˆˆˆPZ ĩ˛˛Âŋ”NQMMMÅ­[ˇ ¯¯œ8q%%%°ˇˇ‡ūûī?€ˇˇ7nßž´´4¨T*tėØ'OžDqq1ėėėĐŦY3ÄÄÄZļl‰ėėlܸqĐącGDGGŖ°°666pqqÁšsįžžžČĪĪĮõë×ÁÁÁ8ū<ōķķaeewwwÉgV­V‹÷;00qqqČÉɁ……Zļl)NÁtuu…žžžä3ģwīÂÔÔžžžâũnŪŧ9ŒŒŒ/ŪīÄÄDÜšsÆÆÆh׎Nœ8!~fÍÍÍÅûíįį‡7n ##Cã~;88ĀÚÚZŧß>>>HOOGzz:ôôôĐĄCņ~7mÚM›6Ell, UĢVČĖĖDjj* S§NˆŠŠBQQėėėāččˆ .ŧŧŧ““#Ūī:āĖ™3(((€ \]]qöėYņs[XXˆäädņ3[Õw„ąą1õáë닛7oJž#Ž?A`oo[[[qÔkëÖ­‘‘‘´´4ņ~GFFB­VŖI“&ppp?ŗ­ZĩBVVnŪŧŠqŋmmmáėėŒķįĪ‹÷;77)))€œ;wųųų°ļļF‹-Äû]“īˆĶ§OZ´h¸ví }ûö¸|ų2˛ŗŗaffÆīˆGü;BĨRĄuë֊úŽPâß#øÁīˆÆøŽ(**?€2ž#”ø÷]˙Ž°ąąAAArrr```ĩZ-Θą°°@NNA€ ‘—— ôīW%%%âžæææČËËCII ôõõadd$ŲWĖĖĖŸŸ/îkllŒÜÜ\€‘‘Hö-((€Z­†žžLLL$ûĒT*q „ŠŠ) Å}MMM‘““ tĄt===ÉžEEE(..†JĨ‚šš9˛ŗŗÅ}õõõ‘ŸŸ_åže÷åū}MLLÄ{x˙ž÷ßÃ˛}  §§'š‡•ŨīŠ÷đūû]ņVvŋËîae÷ģâ=Ŧíũ.ģ/õyŋMLLP\\\éũ600€AĨ÷ģ.ŸŲęî÷ƒ>ŗõyŋ+ŪÃÚŪī}fõõõQPP€Ķ§Ok|GÜ?¸¨ļT‚žŖéęęŠéĶ§ã7ŪÛ>ūøclܸQü—Ëũž|ōIâË/ŋÛvî܉Áƒ#77WãI •ĀruuEff&ŦŦŦęų5ŧˆˆ56o*,UAnŪ…J5 }úôÁöíÛaff†íÛˇ#99&&&¸zõ*,X€N:áß˙Exx8zôčx÷Ũw‘žžŽuëÖĄ[ˇnGDDâããQXXˆwŪyŠŠŠXĩjƏ/+û¸egg‹~CCCX[[Ŗ{÷î8xđ 8€îŨģ‹û>›6mBXXFU×[GTī*Ë#Ƀy$R摀Ōõ—âããáááÁ‘@2ÉÎÎæXÔ ”ķŦŦ,X[[?tíEq#°rssĄ§']šK__%%%UĶšsgüōË/’ļ?ūø!!!•>&ÔØØÆÆÆõĶa¸˙~@õ…Ído $¤vīŪ-Ū§[ˇnáŗĪ>˙(ņ˙”•ņķķ pņâEŦ[ˇNüŋ3:uBHH^~ųe¤ĻĻbΜ9?~|ĩũŸ4i6lØb1ŒH[U–G"’ķH¤Ė#‘2T6C‰H[(îß$ũúõÃüųķņÛoŋ!!!;wîĒ%Kđ˙÷â>3fĖĀˆ#Äí &āęÕĢøß˙ū‡˜˜Ŧ[ˇk׎ŴiĶäx ŽC‡m]ģ–>m°ęī'S¨T6ˆ‹;&ŗĪÍÍÅĉőUüņŒÅŅQeėíí@,z(]5vėXlÛļ oŋũ6>üđÃõ˙Ũwßşū‰?˙ü‹/ŽŅ1DJUY‰HĖ#‘r0ô([ŋ~=llläî€ŌŠlDÚJqŦe˖aĐ A˜8q"|}}1mÚ4ŧöÚkøčŖÄ}RRRÄĩ €Ōyå{öėAxx8đŅGáĢ¯žÂĀåx Žl=”¨›á@Âf觇ãË/ÔĒ*bŲaūü=(..Fhh(ĸĸĸ T*Š‹‹‘‘‘­[ˇJĻZVgÖŦYXŋ~=|||‚-[ļˆë<ˆŸŸzõę…^Ŋz!88¸Ō}ĸĸĸ°fÍqíƒC‡a͚5â\f"ĨķHD˛c‰”ƒy$]˜˜ˆW_}ÎÎÎ022‚››Ū|ķMÜēuKÜĮŨŨK—.•¯“Õ([ĮˆH)Ž€eii‰ĨK—âęÕĢČËËÃå˗ņņĮ‹ ėĨėû§™uëÖ QQQ(((@||<&L˜ĐČ=—Z­b7Ãíŋ€įPĀ¯ė°ÃÎĨŖysÍcėí3:c÷îŨČËËCīŪŊqņâE,_žM›6Œ%Kāââ‚&MšÔ¸GŽÄÆÆbȐ!2dVŦXQ/īq÷îŨ7nŽ; øq㐞ž^/į'Ē/jĩZî.Ņ=Ė#‘r0TŸÔj <ŧtŨßđđŌí†våĘ„„„ ..›7oÆĨK—°råJėßŋ;wFFFFÃwâ>e „׆–Ā&ĒÅ-â.‡ē.$&ˇáKā8úmā*€Šš*n€ú׍8œ6 ))Ĩkcuí čëËÔY"wéŌ%´lŲRîn˜G"%a ¨ŸEÜwėŪ|¸÷`LĨK§|ų%0`@=u´Ī<ķ Ν;‡¸¸8˜ššŠí7n܀——FŒ˜˜>îîî˜2e &NœHHH€‡‡~üņG,_žĮŽÊ+đÔSOaŌ¤IøįŸPXXwww,Z´}ûö­ô}äįįs}jPÔ"îTK%j8ŦšŖYŧÂŊíĢ€ūü‰čžņe@U+ĸ†æāā wˆčæ‘H9˜GĒ;vƒ÷ÁHN.mßļ­aŠXØˇoæĪŸ/)^€ŖŖ#† †ü/^D@@ƏqãÆIöËÍÍÅįŸŽīŋ˙zzz>|8ĻM›†M›6žũö[|øá‡øú믈S§Naܸq077Įȑ#Åķŧ÷Ū{Xŧx1ÂÂÂ`llŒņãĮŖ°°‡‚šš9.\¸đĀ§ Vö3"mÁ–ļK; Ŋ?îj¯Ęöei‡fŨącDĻ˜˜>&œH!˜G"å`ŠŽÔęŌ‘W•Í„Ōu§Nú÷¯˙Ų&/^„ đõõ­ôu___Üž}jĩúúú°´´„ŖŖŖdŸĸĸ"Ŧ\š^^^JŸÄ>oŪ<ņõ>ú‹/ƀ{8\¸pĢV­’°ĻN*î׎]ÃĀŅļm[€§§įßK^^Ū \DJÆ–ļËKnWŗĪí{ûiĄÃ‡ĨĶī'@bbé~Ũģ7Zˇî]ģ´ĒĻĒúđ033‹Wāää„ÔÔT@ZZš¸@|ő[ÅÅÅ°ļļ–œ'$$D˛=eĘŧūúëøã?ĐĢW/ 8íÚĩĢķ{"R"Å-âNĩdęØVŗíŊũˆ¨ÁĩjÕJî.Ņ=Ė#‘r0TW)5ü˙ņ5Ũ¯6Zļl •J… .Túzll,lmmŅ´iĶ*Īq˙Ô=•J%žJJJ”N#ŒŽŽΝ;'>ÄĒŒšššd{ėØą¸rå ^yåœ={!!!XļlY•ũāúW¤ÍXĀŌvö]ĄîmQē`{eTúXö]ŗWDŦŦŦ,šģ@D÷0DĘÁm°ĒYz*āęZē_Cøúë¯QPP€>}úāĐĄCHLLÄŪŊ{Ņģwo4oŪķįĪ¸ģģãĐĄCHNNFzzzĪ?gÎ|úé§øōË/‡ŗgĪ",, K–,yāqS§NÅž}û¨¨(üũ÷ßUŽÕ”ŽÅE¤­XĀŌöOŋo[v(-\ŲxŲĒ´Ũg˜Ė=$"""""zxúúĀ—_–ū~Ģl{éŌú_ĀŊLĢV­ ///ŧôŌKđōōÂøņãŅŖG=zvvv€yķæ!!!^^^°ˇˇ¯ņųĮŽ‹5kÖ`ũúõhÛļ-ēuë†õë×W;K­Vã7Ū€¯¯/ž~úi´nŨ˗/¯Ķ{%R*• Tö‡GKVVŦ­­‘™™ +++šģķđ’ŽĄn€ €vOË˙æČ+"""""R„üü|ÄĮĮÃÃÃãĄ×bÚąŖôi„twu--^Ux8ÉäA9¯kí…#°t@TTTé/z@ €ŖR›°xE$1D$;æ‘H9˜GĒ/ ĀĀ?”ū3>žÅĢšĘÉɑģ DÍ@îPŨ‰ķ˜õ*Ô#ī=ɂˆ× Ræ‘H9˜GĒOúú@÷îr÷B;qi3ŽĀŌļļļĨŋT,`ņ‹‰Hb‰HvĖ#‘r0DĘ``Ā1,¤ŊXĀŌÎÎÎĨŋp‘ėÄ<‘ė˜G"å`‰”ÁĐĐPî.=4°tĀųķįKŠø8°ˆd!摈dĮ<)ķH¤ yyyrwčĄą€ĨK8…ˆˆˆˆˆˆˆt X:ĀËËĢôN!$’˜G"’ķH¤Ė#‘2ËŨĸ‡Æ–ČÍÍ-ũ…S‰d'摈dĮ<)ķH¤ %üīDŌb,`途””Ō_8…Hvb‰HvĖ#‘r0DĘPTT$wˆ Xē„S‰ˆˆˆˆˆHÁF…^xĄÁÎŋvíZ„††6ØųuŲúõëacc#nĪ™3âö´iĶ0eʔÆīØ=,`逐Ō_8…Hvb‰HvĖ#‘r0¤ÍF•J•JCCCxzzbÚ´iČÉÉiĐ놇‡CĨRáΝ;u>W÷îŨĄRŠ`ii)ž•J…âââēw´ PŠTˆŽŽ–´ųå—Xŋ~}ƒ\ŗ  ŗgĪÆ| iĪĘĘÂŦYŗāãã8::ĸW¯^Øąc-ŸŊtŅŠ"•J…]ģvÕÛĩŪ}÷]„……!>>žŪÎY,`é€sįΕūÂ)„D˛ķHD˛c‰”ƒy$m÷ôĶO#%%WŽ\ÁĮŒå˗cÚ´iu.A´pT•qãÆáōåËHII 4ö+,,lĐ~X[[WYpŠ‰ŨŋíÛˇÃÂÂ]ģvÛîÜšƒ.]ēāģīžÃŒ3…C‡áĨ—^Âģīž‹ĖĖĖ‡îËŖÆÁÁĄĄĄXšrĨ,×gKäįį—ūÂ)„D˛ķHD˛c‰”ƒy$mgll GGG¸ēēbčĐĄ6l˜8˛eãƍ ĨĨ%1tčP¤ĻĻŠĮ–¤ÚˇoBBB`llŒÃ‡C,\¸žžž055EûöíąmÛ6ĨŖ—zôč°ĩĩ…JĨ¨QŖ”Ž2š2e `bb‚'žx'Nœ¨ö=˜™™ÁÁÁŽŽŽâ¸ģģãã?ƨQŖ`mmqãÆ(-ųûûÃØØîîîXŧxąä|îîîøä“O0fĖXZZĸE‹XŊzĩøē‡‡ 00*• Ũģw 9…đA÷áA÷¯2[ļlÁķĪ?/i›9s&‘#GÂĪĪŪŪŪ7nĸŖŖaaa¸}û6FŒ[[[˜™™á™gžÁŋÅķ”tÚˇo|}}aaa!6`ßž}011Ņ17eĘtëÖMÜ>räž|ōI˜ššÂÕÕSĻLGķ}÷Ũw°°°\wōäÉđööŽķˆŋĘFôEGGCĨR!!!ĄÆįyūųįąyķæ:õåaą€ĨŦ­­KáB"Ų‰y$"Ų1DĘÁ<’Ž155D/,,ÄG}„Ķ§Oc׎]ˆ‹MŊûîģøôĶOƒvíÚáũ÷ßGXXVŦXķįĪã­ˇŪÂđáÃqđāA¸ēēbûöí€˙ūû)))øōË/Åķlßž6l@TTZļl‰>}ú ##ŖÚ~ëëëWÚžhŅ"´iĶ'OžÄ|€“'ObđāÁxųå—qöėYĖ™3|đÆÔŋŋ#$$§NÂĉņúë¯#66püøqĀ_ũ…””ėØąŖŌk?č><čūUæđáÒ)Ë%%%Ø˛e † gggũ-,,ÄQhŖFBdd$vīŪŖGBôíÛW˛đ}nn.>˙üs|˙ũ÷8tčŽ]ģ&ŽÄëÕĢlllÄ?7PĢÕØēu+† 8{ö,úôéƒāĖ™3øņĮņĪ?˙`Ō¤I€#F oßž6lŠ‹‹ąwī^ŦZĩ ›6m‚ššyĨīšąuė؉‰‰¸zõjã_\ !33S dffĘŨ•‡’““Sú‹Z-Ĩ“ĄKy;EôˆķHD˛c‰”ƒy$A„ŧŧ<á… B^^^ycp° 4oŪø?ÁÁ5î÷ȑ#…ūũû‹ÛB“&M„ÁƒWē˙ņãĮÂŨģwA„„]ģv‰ûdgg &&&‘#G$ĮžúęĢ!C$ĮŨž}[rœĄĄĄ°iĶ&ą­°°Ppvv.\Xå{čÖ­›`hh(˜››‹?˙ûß˙A777á…^ė?tčPĄwīŪ’ļwŪyGđķķˇŨÜ܄áNjÛ%%%‚ƒƒƒ°bÅ A!>>^ œ:uJržŠ÷ŗ6÷ĄâũĢĖíÛˇÂĄC‡Äļ›7o „%K–<đظ¸8€đīŋ˙Šméé邩ŠŠ°uëVA!,,L \ētIÜį›ožš5k&nO™2ExꊧÄí}ûö FFFBFF† ‚đĘ+¯ãĮ—\ûđᾞž˜‹ŒŒ ÁÅÅExũõׅf͚ üņû^Ö¯Šļe?„;w ‚PųįéÔŠS!>>^<—ĩĩĩøú‡~(´oß^rŊ˛úIxxxĨũŠ4į÷û°ĩÍ ¯¤uΞ=‹N:q‘ˆy$"Ų1DĘÁčŌĨ ĻOŸūĀž€ĨĨ%ĸĸĸ4Ú[ĩjUíąĩejj  t4ZccK—°€EDDDDDÚäŪ:LJŋn=°bÅ ÂŲŲ†††€œœ„††"447n„ŊŊ=Ž]ģ†>}úh,†^q XY‘ëˇß~CķæÍ%ûWŲĒ 2‚ T[¤ąļļ†———¸æSU}Ģę|B% +ģeT*•øŪjĸ6÷Ąē)tMš4JĨÂíÛˇÅ6{{{ØÚÚV[ÜĢėŊ•ĩWŧ•ŊߊĮvėØ^^^Ø˛e ^ũuėÜšaaaâë%%%xíĩ×0eʍkĩhŅBüũĐĄCĐ××Įõëב““++Ģö_OO-[ļŦvŸ˛÷TĻâôȚ*›Ējoo_ëcëŠ,āîî^žĄ§WZŧâS‰d!É#Ɋy$Ræ‘Ē)wjÄÜÜŧŌAll,ŌĶĶņŲgŸÁÕÕYƒ÷äįįccc\ģvM˛ĀwEFFFJ×Q*Ķ˛eKáŸūÁĐĄC”!"##1uęÔj¯û âØũũûįŸ$mGŽˇˇw•ëhÕ¤˙•]§ēûPSFFFđķķÅ   ´hķŌK/áûīŋĮ‡~¨ąVNNŒáįį‡ââbDDD K—.€[ˇn!..žžžĩęĮĐĄCąiĶ&¸¸¸@OOĪ>ûŦøZPPΟ?˙ĀbĶ‘#G°páBüōË/˜>}:&OžŒ 6ÔĒ•)+8Ĩ¤¤ˆ#ÂĸŖŖk}žsįÎÁĐĐūūūuîSmqw y„hŲ“9‹Hr<™ˆ*Į<)ķHēĒE‹022²eËpåĘėŪŊ}ôQĩĮYZZbÚ´ixë­ˇ°aÃ\ž|§NÂ7ß|#+ÜÜÜ RŠđë¯ŋ"-- ŲŲŲ077Įë¯ŋŽwŪy{÷îŅ 0nÜ8äææâÕW_­öēU4ēßÛoŋũû÷ãŖ>B\\6l؀¯ŋūZ\°ŧ&`jjŠŊ{÷âæ͛ČĖĖ|¨ûP}úôŅ(ŧ}ōÉ'puuE§NđŨwßá… ¸xņ"Ö­[‡€€dggŖUĢVčßŋ?ƍ‡ūų§OŸÆđáÃŅŧysŠ”Õ6lĸĸĸ0ū| 4H2ĨņŊ÷ŪÃŅŖGņÆo ::/^ÄîŨģ1yōdĀŨģwņĘ+¯`ōäÉxæ™gđÃ?`ëÖ­ø駟j}/îײeK¸ēēbΜ9ˆ‹‹ÃoŋũĻņdɚ8|ø0ēví*N%lL,`逤¤¤ō˛á,`ÉB’G"’ķH¤Ė#é*{{{Ŧ_ŋ?ũôüüüđŲgŸáķĪ?¯Ņą}ôfĪžO?ũžžžčĶ§~ųåxxxš7oŽšsįbúôéhÖŦ™ø¤ēĪ>û Ä+¯ŧ‚   \ēt ûöíGÕ<ČũĶĢ„­[ˇb˖-hĶĻ fĪžyķæUútÅĒāĢ¯žÂĒUĢāėė\e!¨ēûPãƍÞ={$Å2[[[;v ÇĮĮŒĀĀ@tíÚ›7oÆĸE‹Ä§¤†……!88Ī=÷:wî A°gĪiƒÕiÕĒ:tč€3gΈO,ĶŽ];øNNN€7ß|æææøä“OūūūX°`&L˜€ä:ŽghhˆÍ›7#66íÛˇĮ‚ đņĮ×ú<›7oƸqãęԗ‡ĨjZ‚ÕaYYY°ļļFfffĩsK•(""ĸ|ą9  N’ĩ_D"I‰HVĖ#‘r0ųųųˆ‡‡‡‡dT 5žėėėJ×ĀŌ%ƒF`` fĖ˜!wWtÎoŋũ†wŪyǵAå+R=(įu­Ŋp– *ßāB"YIōHD˛b‰”ƒy$R333šģĐā-Z¤ķE:šäää ,,ŦĘâUCcKÄÆƖop !‘Ŧ$y$"Y1DĘÁ<)C~~žÜ]hpnnnâšRTŋ,ëhZ°t@nnnųFŲ,Î %’…$D$+æ‘H9˜G"e(á@Ōb,`éÉđHN!$’‡+)ķH¤Ė#‘2čëëËŨĸ‡Æ–đōō*ßāB"YIōHD˛b‰”ƒy¤Šø1ųËŨŌq ™o°tĀéĶ§Ë78…HV’<‘Ŧ˜G"å` (ũSXX(sO]œÎK ­ė3fhhXīį–géxj8œBHDDDDD d``333¤ĨĨÁĐĐzzOŅØ d{‚é6A››‹ÔÔTØØØ4ČtU~ru@‹-Ę78…HV’<‘Ŧ˜G"å` T*œœœĢW¯ĘŨG’Z­æ:XÔ lllāččØ įfK×p !)”‘‘ZĩjÅi„2IMM…ƒƒƒÜŨ ehhØ R°tĀĩk×āääTēÁ)„D˛’䑈dÅ<)ķHéééÁÄÄDîn<’RRR8"’´'ëN!$"""""""Öhßž}ų§ÉJ’G"’ķH¤Ė#‘20‹¤ÍXĀŌ—/_.ßāB"YIōHD˛b‰”ƒy$Rf‘´ X: ;;ģ|ƒS‰d%É#Ɋy$Ræ‘H˜EŌf,`é33ŗō N!$’•$D$+æ‘H9˜G"e`I›ą€Ĩ|||Ę78…HV’<‘Ŧ˜G"å`‰”Y$mÆ–ˆŠŠ*ßāB"YIōHD˛b‰”ƒy$Rf‘´ Xē†S‰ˆˆˆˆˆˆHĮ°€Ĩ\\\Ę78…HV’<‘Ŧ˜G"å`‰”Y$mÆ–000(ßāB"YIōHD˛b‰”ƒy$Rf‘´ X: !!Ą|ƒS‰d%É#Ɋy$Ræ‘H˜EŌf,`éN!$"""""""ÖhÛļmų§ÉJ’G"’ķH¤Ė#‘20‹¤ÍXĀŌ׎]+ßāB"YIōHD˛b‰”ƒy$Rf‘´ X: 33ŗ|ƒS‰d%É#Ɋy$Ræ‘H˜EŌf,`é“ō N!$’•$D$+æ‘H9˜G"e`I›ą€ĨÚ´iSžÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3°t@dddų§ÉJ’G"’ķH¤Ė#‘20‹¤ÍXĀŌ5eS9‹ˆˆˆˆˆˆˆt X:ĀÉÉŠ|C¯Â)‹XDN’G"’ķH¤Ė#‘20‹¤ÍXĀŌfffå XœFHÔč$y$"Y1DĘÁ<)ŗHڌ,pųōåō˛)„ XD2ä‘ˆdÅ<)ķH¤ Ė"i3°t §‘ŽaKøûû—op !‘Ŧ$y$"Y1DĘÁ<)ŗHڌ,pũúõō N!$’•$D$+æ‘H9˜G"e`I›ą€Ĩnßž]žÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3°t€ĄĄaų§ÉJ’G"’ķH¤Ė#‘20‹¤ÍT‚Āa:YYY°ļļFff&ŦŦŦäîNŨôéüņGéīwîÖÖ˛v‡ˆˆˆˆˆˆˆ¨ŽĩŽĀŌåœBH$+I‰HVĖ#‘r0DĘĀ,’6cK×p !é°t@ŗfÍĘ7øB"YIōHD˛b‰”ƒy$Rf‘´ X:@2w”S‰dĨõëčéæ‘H9˜G"e`I›ą€Ĩ.^ŧXžÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3Å°ÜŨŨĄRŠ4~ŪxãJ÷¯t˙ØØØFîšBp !éš;pŋ'N@­V‹ÛįΝCīŪŊņâ‹/>đ¸˙ūûO2ŌŪŪžÁú¨4žžžåœBH$+I‰HVĖ#‘r0DĘĀ,’6S\ëūÂĶgŸ}///tëÖíĮ988ĀÆÆĻ{Ļ\ŠŠŠåÅ;N!$’•$D$+æ‘H9˜G"e`I›)n aE………ظq#ƌUÅŠq• „““zöė‰4R•áÖ­[åœBH$+I‰HVĖ#‘r0DĘĀ,’6SÜŦŠvíڅ;wî`Ô¨QUîãää„ÕĢW#88øūûīŅŗgO„‡‡ãÉ'ŸŦô˜‚‚ˆÛYYYõŨõFĨ¯¯_žÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3• (ˇĘŅ§Oá—_~ŠÕqũúõƒJĨÂîŨģ+}}Μ9˜;wŽFûūũûannŽ   ÄÄÄ //–––đđđĀ™3gnnn())Abb" —.]Bvv6ĖÍÍáííS§N\\\ ¯¯ĢW¯Úĩk‡„„deeÁÄÄūūū8yō$ĀŲŲ&&&¸rå  M›6HJJ;w`dd„€€?~āčč \ēt @é<æ›7o"##­>üvûöĸwė€Uûö°ĩĩE\\ uëÖČČČ@ZZôôôĐĄCDFFB­VŖI“&ppp@LL  UĢVČĘĘÂ͛7:uBTTŠŠŠ`kk gggœ?āåå…ÜÜ\¤¤¤BBBpîÜ9äįįÃÚÚ-Z´ĀŲŗg”.Ô_\\Œ¤¤$@PPbcc‘›› xyyáôéĶ€-ZŽ]ģhßž=._žŒėėl˜™™ÁĮĮQQQâũ600@BB mÛļ¸ví233abb‚6mÚ 22@iáĶĖĖ —/_øûûãúõë¸}û6 „ˆˆ@ŗfÍ`ee%>ąÃ×׊ŠŠ¸uëôõõ‚'N ¤¤ööö°ŗŗÃ˙ũđööÆíÛˇ‘––•J…Ž;âäɓ(..†š5k&Ūī–-[";;7nÜtėØŅŅŅ(,,„ \\\pîÜ9€§§'ōķķqũúu@pp0Ο?üü|XYYÁŨŨ]ō™UĢÕâũ D\\rrr`aa–-["::āęę ===Ég6>>wīŪ…ŠŠ)|}}ÅûŨŧys!>>^ŧ߉‰‰¸sįŒŅŽ];œ8qBüĖš››‹÷ÛĪĪ7nÜ@FF†Æũvpp€ĩĩĩxŋ}||žžŽôôtņ3[vŋ›6mŠĻM›ŠmhÕĒ233‘ššĒņ™ĩŗŗƒŖŖ#.\¸ ~fsrrÄûŨĄCœ9s°ąąĢĢĢø™õđđ@aa!’““ÅĪŦļ~G 88Į‡ °ˇˇįwŋ#øÁī~G€ßüŽāwŋ#øÁī~G4ôw„““ÜÜܐ™™ųPSY[Āēzõ*<==ącĮôīßŋVĮΟ?7nÃzŋĘF`šēē>ôM”Û‰'ĐĄC‡ŌaÀ~(ũũŌ%ĀËKžŽ=‚$y$"Y1DĘÁ<)ŗHrĘĘʂĩĩõC×^;…0,, xöŲgk}ėŠS§āääTåëÆÆÆ066ŽK÷Ĩ¤âZWœBH$ĢŽ=G¤Ė#‘r0DĘĀ,’6SdĢ¤¤aaa9r$ ¤]œ1c’““ņŨwß–.] wwwøûû‹‹žoßžÛˇo—Ŗ벐<š‘O!$’ÕũOR%"ų0DĘÁ<)ŗHÚL‘Ŧŋūú ׎]Ø1c4^KIIį(ĨO*œ6m’““ajj üöÛočÛˇocvYVvvvå| !‘Ŧ$y$"Y1DĘÁ<)ŗHÚL¯ú]_hh(A€ˇˇˇÆkëׯGxx¸¸ũîģīâŌĨKČËËCFF>üH¯ˆ‹øāB"™IōHD˛b‰”ƒy$Rf‘´™" XTœBHDDDDDDD:†, ŠÆ)„D˛Ēlä(Ƀy$Ræ‘H˜EŌf,`é€Ûˇo—op !‘Ŧ$y$"Y1DĘÁ<)ŗHڌ,––VžÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3°t€Ēâ´AN!$’•$D$+æ‘H9˜G"e`I›ą€Ĩ:vėXžÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3°tĀɓ'Ë78…HV’<‘Ŧ˜G"å`‰”Y$mÆ–(...ßāB"YIōHD˛b‰”ƒy$Rf‘´ X:ĀÎÎŽ|ƒS‰d%É#Ɋy$Ræ‘H˜EŌf,`é€f͚•op !‘Ŧ$y$"Y1DĘÁ<)ŗHڌ,SžÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3°t §‘ŽaK´lŲ˛|ƒS‰d%É#Ɋy$Ræ‘H˜EŌf,`é€ėėėō N!$’•$D$+æ‘H9˜G"e`I›ą€ĨnܸQžÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3°t §‘ŽaKtėØą|ƒS‰d%É#Ɋy$Ræ‘H˜EŌf,`é€čččō N!$’•$D$+æ‘H9˜G"e`I›ą€Ĩ Ë78…HV’<‘Ŧ˜G"å`‰”Y$mÆ–°ąą)ßāB"YIōHD˛b‰”ƒy$Rf‘´ X:ĀÅÅĨ|ƒS‰d%É#Ɋy$Ræ‘H˜EŌf,`é€sįΕop !‘Ŧ$y$"Y1DĘÁ<)ŗHڌ,]Ã)„DDDDDDD¤cXĀŌžžžåœBH$+I‰HVĖ#‘r0DĘĀ,’6cKäįį—op !‘Ŧ$y$"Y1DĘÁ<)ŗHڌ,pũúõō N!$’•$D$+æ‘H9˜G"e`I›ą€Ĩk8…ˆˆˆˆˆˆˆt X: 88¸|ƒS‰d%É#Ɋy$Ræ‘H˜EŌf,`é€ķįĪ—op !‘Ŧ$y$"Y1DĘÁ<)ŗHڌ,På"îœBHÔč¸0&‘r0DĘÁ<)ŗHڌ,`eeUžÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3°t€ģģ{ų§ÉJ’G"’ķH¤Ė#‘20‹¤ÍXĀŌgΜ)ßā,"YIōHD˛b‰”ƒy$Rf‘´ Xē†k`‘ŽaK¸šš•op !‘Ŧ$y$"Y1DĘÁ<)ŗHڌ, VĢË78…HV’<‘Ŧ˜G"å`‰”Y$mÆ–HJJ*ßāB"YIōHD˛b‰”ƒy$Rf‘´ Xē†S‰ˆˆˆˆˆˆHĮ°€ĨË78…HV’<‘Ŧ˜G"å`‰”Y$mÆ–ˆ‹‹+ßāB"YIōHD˛b‰”ƒy$Rf‘´ X: ''§|ƒS‰d%É#Ɋy$Ræ‘H˜EŌf,`é ‹ō N!$’•$D$+æ‘H9˜G"e`I›ą€ĨZļlYžÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3°t@tttų§ÉJ’G"’ķH¤Ė#‘20‹¤ÍXĀŌ5œBHDDDDDDD:†,āęęZžÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3°t€^Åĸ§ÉJ’G"’ķH¤Ė#‘20‹¤ÍøéÕW¯^-ßāB"YIōHD˛b‰”ƒy$Rf‘´ Xē†S‰ˆˆˆˆˆˆHĮ°€ĨÚĩkWžÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3°t@|||ų§ÉJ’G"’ķH¤Ė#‘20‹¤ÍXĀŌwīŪ-ßāB"YIōHD˛b‰”ƒy$Rf‘´ X:ĀÔÔ´|ƒS‰d%É#Ɋy$Ræ‘H˜EŌf,`é__ßō N!$’•$D$+æ‘H9˜G"e`I›ą€ĨĸĸĸĘ78…HV’<‘Ŧ˜G"å`‰”Y$mÆ–ŽáB"""""""Ō1,`é€æ͛—op !‘Ŧ$y$"Y1DĘÁ<)ŗHڌ,`ddTžÁ)„D˛’䑈dÅ<)ķH¤ Ė"i3°t@|||ų§ÉJ’G"’ķH¤Ė#‘20‹¤ÍXĀŌ5œBHDDDDDDD:†,ĐļmÛō N!$’•$D$+æ‘H9˜G"e`I›ą€ĨË78…HV’<‘Ŧ˜G"å`‰”Y$mÆ–¸sįNų§ÉJ’G"’ķH¤Ė#‘20‹¤ÍXĀŌÆÆÆåœBH$+I‰HVĖ#‘r0DĘĀ,’6cK´k׎|ƒ#°ˆd%É#Ɋy$Ræ‘H˜EŌf,`é€'N”op ,"YIōHD˛b‰”ƒy$Rf‘´ Xē†S‰ˆˆˆˆˆˆHĮ°€ĨË78…HV’<‘Ŧ˜G"å`‰”Y$mÆ–077/ßāB"YIōHD˛b‰”ƒy$Rf‘´ X:āōåËåœBH$+I‰HVĖ#‘r0DĘĀ,’6cK×p !é°t€ŸŸ_ų§ÉJ’G"’ķH¤Ė#‘20‹¤ÍXĀŌ7nÜ(ßāB"YIōHD˛b‰”ƒy$Rf‘´™â XîîîPŠT?oŧņF•Į}Äīˆk׎ÚˇoË—/#;;fffđņņyāwÄĩkא™™ ´iĶ‘‘‘'''~GčĀwDNN233õĄÄŋGđ;‚ߍņ‘••…ĸĸ"E}G(ņīüŽāwDCGÜž}ŠûŽPâß#øQ˙ß•ÕgjCŅSûôé###üōË/UîãííŅŖGcƌbÛŋ˙ū‹'žx)))pttÔ8Ļ  âvVV\]]ĩv a||<<<ųä\ŧxķįĪo ^Ũ—G"’ķH¤Ė#‘20‹¤ÍYĀ*))AXXFŽ é ą3f`Ĉâö„ põęUüī˙CLL Ö­[‡ĩk×bÚ´iŨme¨8ĢšŲĄįΝÃčŅŖņķĪ?ÃŲŲ@iA0:::uÂöíÛŅĄC¨Õjck3ķ´K—.@§Nj|ĖũÜÜÜ0pā@˜ššÂĖĖ  įāVôŨw߉ Dž…‰íŸū9vė؁ß˙fffŨ"""""""j|ŠœBø×_áÚĩk3fŒÆk)))â"{@é‚g{öėÁ[oŊ…ožųÎÎÎøęĢ¯0pāĀÆė˛Ŧ|||Ę7î°ÔĐÃá4¤lœœ€Ž]Šŗ풓“ņ / ,, ūūūb{||<š7oŽÁƒã駟†ƒƒ˛ŗŗaee…ĖĖLqŋΝ;ãÕW_E\\ŧŊŊąf͚*ûwäȑ:ŋĮĄC‡b÷îŨ5jAĀŸū‰'Ÿ|Rcŋ#FH œ°dÉlŪŧũõlllęÜĸ‘䑈dÅ<)ķH¤ Ė"i3EŽĀ … đööÖxmũúõ—´uëÖ QQQ(((@||<&L˜ĐH=U†ôôôō • ;đpGzûC‡=zîî@ÅåÄÖŦYƒ´´4ŧõÖ[âˆĨŨģw#<<ÁÁÁĀã?ŽE‹ÁÚÚ={öDNNÚˇo &ĀÁÁĢW¯Fŋ~ũĐĨKčéÕũŖ´qãF¸¸¸ā§Ÿ~Â|ņ /ŋü2œáīīöíÛÃÉÉŠŌāī—””„ˇß~wîÜA=ę<Œ¨:’<‘Ŧ˜G"å`‰”Y$mĻč§6–ēŽ„/ˇˆˆą(ŗãû aŠŌ?ÔōĸRŲŌXÛļ•>œˆFÅ<‘ŧ˜G"å`‰”Y$9ÕĩöĸČXT;eŖŸÔjāÍéšÅ+ |9ŦŠSK÷#ĸ†QŖ‰¨~0DĘÁ<)ŗHڌŸ^ĐĄCĀáÃ@Ōu=TõĮ*@bb TpwwoŧPŠTPU|Bb5ēwī•J%Nm°ļV­Z===<ņĸsįžųæxyyÁÔÔļļļčŨģ7bbbáááPŠTčĐĄF [[[´nŨ€QŖF‰#­Ę~F2d† ōĐī1>>ķįĪĮØącņŅGĸŖŖú|D Ĩ,D$?æ‘H9˜G"e`I›ą€Ĩš6m čÚpi.@…Ęŋ”T*i‘kųōåxũõ×Ņ§OüņĮ°ąąŖŖ#Ū|ķM|ų嗘8q"8€ącĮJÎ  0qqqxīŊ÷¯ŋū:6oŪŒ 6ĀÁÁĐĨK—jûŸôôt¤§§#33ŗŌ}ŒÅßwīŪ čÕĢWĩį&jley$"ų1DĘÁ<)ŗHڌ,Pö%¤¯|š¤4ŠXeËO͞]úĪäädŧņÆ ÅîŨģaff¸uë>ûė3ŧöÚkøä“O VĢ%9—ŸŸ,X€éĶ§.]ēčÔŠ^|ņEüōË/HMMŜ9s0~üøjû?iŌ$ØÛÛÃŪŪũû÷¯r?AđöÛocŨēu0`Ū}÷ŨjĪMÔØø—"å`‰”ƒy$Rf‘´ X: 66Vü}Āļaš#Y˛‹ °mđôĶĨÛĻĻĻ°ąąÁącĮpúôi@nn.&NœˆôôtŦ[ˇüņŒ‘ŸŸ/9—ŊŊ=ĀĐĐP\\Z4cĮŽÅļmÛđöÛoãÃ?ŦQ˙ß}÷]üųįŸøķĪ?ąxņâJ÷)((ĀË/ŋŒ%K–āÕW_ÅÖ­[Ą¯¯_Ŗķ5ĻŠy$"y1DĘÁ<)ŗHÚĖ@îP=*Qi1 ķNôŋũ3L@ĘĖoāäT:ŊP_HH(ŨÕÎΛ7oFīŪŊŠŋūú ­[ˇ†JĨBqq1222päČÔøōŗfÍÂúõëáãバlŲ˛čÔŠĶķķķƒŸŸß÷ ÅĄC‡Œ^Ŋz᧟~‚šš9úõëWãū‘vâ,ĐĒU+ v0ÜhĶ 8觖 ģĮ* ܄îŨK‹W÷ëÜš3vīŪŧŧ<ôîŨ/^ÄōåËŅ´iS,Y˛...hŌ¤IûqäČĨUũ˛EÛWŦXQ/īņĐĄC€“'OŠįžĩjÕJî.Ņ=Ė#‘r0DĘĀ,’6S ‚ ČŨ šeeeÁÚÚ™™™°˛˛’ģ;ĩ–zč 8Œüp@Å?M7ŋo|†ÉĶ9ĸGL||<<<<äîy$Ræ‘H˜E’S]k/ĨíJÔ°ûúÍâîm_0obéôB"jpŠŠŠrwˆîa‰”ƒy$Rf‘´ XÚ.í0 öįh¯Ęöei‡ŗWDDDDDDDDõ†,m——ÜŽfŸÛ÷ö#ĸWŨC ˆ¨ņ0DĘÁ<)ŗHڌ,mgęØVŗíŊũˆ¨ÁEEEÉŨ"ē‡y$Ræ‘H˜EŌf,`i;ûŽ(îi^ē`{eTúXö]ŗW•wEĨBvvvĪŗtéRÉÜí9sæ`Ú´i€ŨģwãwŪŠŅyfΜ ___´oß;vÄß˙]įžÉŨ"ē‡y$Ræ‘H˜EŌf,`i;=}¤Ŋ>ˇôiƒ÷ąĘžB8{9 §ßø}{HÅÅÅ|ũūVEĪ?˙<-ZTŖëtíÚQQQ8}ú4žũö[ 8ųųųĩî/QEvvvrwˆîa‰”ƒy$Rf‘´ X:Ā"d<đûFāe+ĀĨ…+;Īę•ļû “š‡å>˙üs<ūøãđööÆæ͛Åv•J…Å‹Ŗ{÷î˜1cnŪŧ‰˙ûŋ˙CÛļmŅĻMŦ^Ŋ0oŪ<\ŋ~ƒ B@@ĸŖŖ%į_ŋ~= TŖž<ķĖ3055´mÛjĩéééõķFé‘åčč(wˆčæ‘H9˜G"e`I›ą€Ĩ.\¸PZ¤Ú˜\8t7šˆo­¨âPZ¨ú÷ßąwī^Lž<‰‰‰âkĮĸE‹0eĘøøøāėŲŗøûīŋņŅGáøņã˜={6œąmÛ6DGG#  ^ú///¸¸¸ÔËųčŅuášģ@D÷0DĘÁ<)ŗHڌ,]ĸ§4ëD1ԂÜ=Ō0vėX€§§'žxâ >|X|mĖ˜1âīũõŪx㠀ƒƒ €ũû÷7HŸöīߏšsįb˖- r~"""""""Ē°t€———´Auo1Ŧ’’ÆīL-ŠTå wYXXTųZeÛĩqá @,ŒĀÁƒ1zôhüōË/hŨēõCŸŸ¨ŒF‰H6Ė#‘r0DĘĀ,’6cKäääHôîũą 3K­ÂÁ͛K˙ŠVWŊīēuë øįŸđÄOTē_¯^ŊÄu¯ŌŌŌ°sįN<õÔS+++dffÖĒ~~~ˆŽŽFtt4žųæĀĄC‡đĘ+¯āįŸFûöíku>ĸĒh䑈dÃ<)ķH¤ Ė"i3°tĀ7¤ eŦFĩcāîôč ZúOw÷ŌöĘãņĮGhh(–-[WW×J÷ûęĢ¯pæĖ´k×=zôĀŦYŗĐącGĀ”)S0zôčJq¯W_}âšpöėŲ‡>PIuÜõë×ŅŖG:Ÿ'''ŖGFÛļmŅēukLŸ>B#áIw=jy$R2æ‘H˜EŌf*˙…€ŦŦ,X[[#33VVVrw§Ö"""ĐŠS§ō - đđŽ\i°ëîØ ¤9ĐĢlĻßļmĀ€ vy"EŌČ#ÕČŦYŗpũúuŦ[ˇÅÅÅxîšį0vėXŧøâ‹rw´ķH¤Ė#‘20‹$§ēÖ^8KtčĐAÚĐSÕjāÍ7+ŋDYÛÔŠžNH¤‹4ō¨eļoßâã?†JĨBvv6öėŲ#ŽT €‰‰ 6l؀„„4mÚT<^ĨRaΜ9xüņĮáííÍ›7×čē§OŸÆ3Ī<•JCCC„††âûīŋo¨ˇImĪ#‘.a‰”Y$mÆ–8s挴ĄĻ> $%Uũē ‰‰Ĩû=J4ō¨ERSS1~üxüōË/8uę”äÁ }ûöב›6m|}}1 Š!–*• ˙ūû/öîŨ‹É“'#11ąÚkwčĐ[ˇnEaa!îŪŊ‹;w"!!ĄžŪ=ĸ´9Dē†y$Rf‘´ X:   @ÚĐO!LIŠßũˆt…FĩČącĮ„V­ZF­ąĪ0gÎüöÛo°´´ŦôØë~*āęZēŅŖ¤Ē'kĘŠĻO }ėąĮpōäI\ēt °aÃņĩäädŧđ  ƒŋŋ˙¯ˇnŨ:@BBūųį<ņÄû9rŅŅŅˆˆˆPē cnn. >>+VŦĀÛoŋũī˜¨”ķHô¨b‰”Y$mÆ–8{öŦ´ĄĻęë_~)ŊÜũ—_ēTs„‘ŽĶČŖĖʞz˙šuÉÉĨí‹X͚5Ãʕ+ņėŗĪĸK—.ČÉɁĄĄ!ĖĖĖ°fͤĨĨá­ˇŪrßŊ{wĨ×466Æã?ŽĐĐP,[ļŦFQērå āįį‡ūũûã‹/ž@@@@Ū9‘ōōHô(c‰”Y$mĻ„œgĻ%ęú(Gši< ÕĶˆ€›7ôÚ;v”>°âģē–¯ĒXߙH§)éŅÄjuéHĢĒ¸ R•Ž¤Œ//6ßŊ{W\Û*,, k׎Å?˙üSãkĒT*ÜŊ{W˛<‘\””GĸGķH¤ Ė"ÉŠŽĩŽĨ<<<¤ 0…°Ė€@˙ūÕ¯­Cô¨ĐČŖŒjķ´ĐîŨKÛžúę+üôĶO(..†žũöÛFé+QCPR‰uĖ#‘20‹¤ÍXĀŌ………Ō†F˜BX‘ž~ųü=ę4ō(Ŗ‡yZčŦYŗ0kÖŦ‡ž&õ’’()D:æ‘H˜EŌf\K$''Ká)„DT9<ʈO ĨG’ōHô¨c‰”Y$mÆ–.jÄ)„D¤\|Z(é °t@PP´Ą‘§Q9<ʈO ĨG’ōHô¨c‰”Y$mÆ–ˆ‰‰‘6p !‘l4ō(ŗ€mۀæÍĨí..Ĩí|Z(é2Ĩå‘čQÆ<)ŗHڌ‹¸ë€ŧŧ-”EJÍ#ŅŖˆy$Rf‘´§ęi§ÉF#D$æ‘H9˜G"e`I›ą€ĨΜ9#māB"Ųh䑈dÃ<)ķH¤ Ė"i3°t§‘aK¸ššI8…H6y$"Ų0DĘÁ<)ŗHڌ,Pr˙H+N!$’F‰H6Ė#‘r0DĘĀ,’6cK$&&JĘĻ…EÔČ4ōHD˛a‰”ƒy$Rf‘´ XēH¯Â+ XDDDDDDD¤åXĀŌŌ†Š,%jTy$"Ų0DĘÁ<)ŗHڌ,péŌ%iCÅ)„,`5*<‘l˜G"å`‰”Y$mÆ–ČÎΖ6p !‘l4ōHD˛a‰”ƒy$Rf‘´ X:ĀÜÜ\ÚĀ)„D˛ŅČ#Ɇy$Ræ‘H˜EŌf,`éoooi§ÉF#D$æ‘H9˜G"e`I›ą€ĨN:%māB"Ųh䑈dÃ<)ķH¤ Ė"i3°t§‘aK¸¸¸H8…H6y$"Ų0DĘÁ<)ŗHڌ, ¯¯/māB"Ųh䑈dÃ<)ķH¤ Ė"i3°tĀÕĢWĨ œBH$<‘l˜G"å`‰”Y$mÆ–.âB"""""""Ō!,`é€víÚI8…H6y$"Ų0DĘÁ<)ŗHڌ, māB"Ųh䑈dÃ<)ķH¤ Ė"i3°t@VV–´S‰dŖ‘G"’ ķH¤Ė#‘20‹¤ÍXĀŌ&&&ŌN!$’F‰H6Ė#‘r0DĘĀ,’6cKøûûK8…H6y$"Ų0DĘÁ<)ŗHڌ,pōäIi§ÉF#D$æ‘H9˜G"e`I›ą€Ĩ‹8‹ˆˆˆˆˆˆˆtˆA}œ¤  ĮGBBrssaooĀĀ@xxxÔĮéŠÎÎÎŌŽE$<‘l˜G"å`‰”Y$mV§֑#G°lŲ2ėÚĩ ………°ąąŠŠ)222PPPOOOŒ?&L€ĨĨe}õ™îŖą§Ɇ c)ķH¤Ė#‘20‹¤Íz a˙ūũ1hĐ 4oŪûöíÃŨģwqëÖ-$%%!77/^Äûīŋũû÷ÃÛÛūųg}ö›*¸r労S‰dŖ‘G"’ ķH¤Ė#‘20‹¤ÍzVhh(~úé'Uúē§§'<==1räHœ?ׯ_čNR-q !é• °Â‘••kkkdffÂĘĘJîîÔZNNĖÍÍËFÖ¯/ũũÂĀ×W–~=Š4ōHD˛a‰”ƒy$Rf‘äT×ÚKŊ<…011IIIâöņãĮ1uęTŦ^Ŋē>NOÕ¨xīp !‘Œ4ōHD˛a‰”ƒy$Rf‘´YŊ°†ŠnܸŪŊ{ãøņã˜9s&æ͛W— ¸sįŽ´S‰dŖ‘G"’ ķH¤Ė#‘20‹¤ÍęĨ€uîÜ9tėØ°uëV´iĶGŽÁ?ü€õeSŲ¨ÁhŦCƧÉĻĒu‰¨ņ1DĘÁ<)ŗHÚŦ^ XEEE066üõ×_xūųį>>>HIIŠõų’““1|øp4iŌfffĀɓ'ĢÜ?<<*•Jã'66öáŪ– 6p !‘l4ōHD˛a‰”ƒy$Rf‘´YŊ°üũũąråJ>|ūų'ž~úiĀõë×Ņ¤I“ZëöíÛxüņĮahhˆß˙.\ĀâŋaccSíą˙ũ÷RRRğV­Z=ĖÛŅ:Į—6p !‘l4ōHD˛a‰”ƒy$Rf‘´™A}œdÁ‚øŋ˙û?,Z´#GŽDûöíģwī§Öæ\ŽŽŽ ÛÜŨŨktŦƒƒC ]:S‰ˆˆˆˆˆˆH‡ÔËŦîŨģ#==éééXˇnØ>~üxŦ\š˛VįÚŊ{7BBBđâ‹/ÂÁÁøöÛoktl`` œœœĐŗgOqQųGŖŖŖ´S‰dŖ‘G"’ ķH¤Ė#‘20‹¤ÍęĨ€‚ āäɓXĩjîŪŊ  t833ŗZįʕ+XąbZĩj…}ûöa„ ˜2e žûîģ*qrrÂęÕĢą}ûvėØą­[ˇFĪž=qčĐĄJ÷/((@VV–äG›YXXH8…H6y$"Ų0DĘÁ<)ŗHÚL%u¯p\ŊzO?ũ4Ž]ģ†‚‚ÄÅÅÁĶĶS§NE~~~­Fa!$$GŽÛĻL™‚'NāčŅŖ5>Oŋ~ũ RŠ°{÷n×æĖ™ƒšsįj´īßŋæææ BLL ōōō`ii œ9sāæ憒’$&&(]īŌĨKČÎΆšš9ŧŊŊqęÔ)€‹‹ ôõõqõęU@ģv퐐€ŦŦ,˜˜˜Āßß_\œŪŲŲ&&&¸rå  M›6HJJ;w`dd„€€qž˛ŖŖ#,,,péŌ%@qq1‘‘˙=đ嗀ämÛ`öÔSˆ‹‹´nŨHKKƒžž:tč€ČČH¨Õj4iŌˆ‰‰´jÕ YYY¸yķ& S§NˆŠŠBQQlmmáėėŒķįĪŧŧŧ››+.Ú‚sįÎ!??ÖÖÖhŅĸΞ=  tJhqq1’’’AAAˆEnn.,,,āåå…Ķ§OZ´h¸ví }ûö¸|ų2˛ŗŗaffDEE‰÷ÛĀĀ €ļmÛâÚĩkČĖĖ„‰‰ Ú´iƒČČHĨEO333\ž|@é:nׯ_ĮíÛˇahhˆ   DDDš5k+++\ŧxāëë‹ÔÔTÜēu úúú Á‰'PRR{{{ØŲŲáŋ˙ūx{{ãöíÛHKKƒJĨBĮŽqōäIÃÎÎ͚5īw˖-‘7n:vėˆččhÂÆÆ...8wîĀĶĶųųų¸~ũ: 88įĪŸG~~>ŦŦŦāîî.ųĖĒÕjņ~"..999°°°@˖- puu…žžžä3ģwīÂÔÔžžžâũnŪŧ9ŒŒŒ/ŪīÄÄDÜšsÆÆÆh׎Nœ8!~fÍÍÍÅûíįį‡7n ##Cã~;88ĀÚÚZŧß>>>âĪ˛ĪlŲũnÚ´)š6m*>°ĄUĢVČĖĖDjjĒÆgÖÎÎŽŽŽ¸pá‚ø™ÍÉÉīw‡pæĖĀÆÆŽŽŽâgÖÃÃ………HNN?ŗU}GäįįŖUĢVŠúŽđõõÅ͛7Ëŋ#‚ƒqüøq‚{{{ØÚÚō;‚ß:ų‘““ƒāā`E}G(ņīüŽāwDc|GdeeĄ{÷îŠúŽPâß#øÁīˆ†ūŽˆ‰‰­­­âž#”ø÷~GÔ˙w„““ÜÜܐ™™ +++ÔVŊ°^xáXZZbíÚĩhŌ¤ NŸ> OOOüđC˜ššbÕĒUظq#ąhŅĸzę!QŨh䑈dÃ<)ķH¤ Ė"iŗzõÅ_āāÁƒđķķC~~>† www$''cÁ‚ĩ>ßsĪ=‡ŗgĪ"??1117nœäõõë׋#rāŨwßÅĨK———‡ŒŒ >|ø‘)^€Á}uȇ˜B¸jÕ*čééá‰'žĀ;wđÍ7ßĀËË ĻĻĻ°ĩĩEīŪŊÅõÄÂÃÃĄRŠĐĄCŒ=ļļļhŨēĩ¸0Ū¨QŖđVe?ŖF 2C† yč÷:gÎ :={ö„ÔjDQCĶČ#Ɇy$Ræ‘H˜EŌfõōéuvvFtt4ļlŲ‚“'Oĸ¤¤¯žú*† SSĶú¸=@pp°´ĄS+Zž|9&Mš„>}ú`ûöí033ƒŖŖ#Ū|ķM˜˜˜āęÕĢX°`Ǝ‹˙ũW<.22O=õ €uëÖáŊ÷ŪCxx8^ũu<ũôĶ(,,Ä;īŧƒÔÔTtŠÁBōŲŲŲČĪĪÂÚÚēŌũzöė)>-ĄgĪž˜5kVĩį&j,y$"Ų0DĘÁ<)ŗHÚŦ^F`:t†††=z4žūúk,_žcĮŽ…ĄĄ!:T— ({œĨ¨S“““ņÆo 44ģwī†™™āÖ­[øėŗĪđÚk¯á“O>Z­iZÆĪĪ ,ĀôéĶ@|Œf§Nđâ‹/â—_~Ajj*æĖ™ƒņãĮWû>&Mš{{{ØÛÛŖ˙ūUîˇ|ųrėÜš}úôÁūũûV횉‹F‰H6Ė#‘r0DĘĀ,’6Ģ—X=zô@JJ $홙™čŅŖGĨ ŧSũîŸ&X‹)„ĻĻĻĐ××ĮącĮpúôi„„„ 77'N„žžÖ­[ôë×OUÆŪŪ@éh)(..û3vėXlÛļ oŋũ6>üđÃŊwß}ÇØÚÚVš_ĮŽ-Z´Āž}û°víZL:ĩF× jhy$"Ų0DĘÁ<)ŗHÚŦ^ X‚ TēŅ­[ˇ`nn^— (+$‰Tā Ā@Á) ä)@OŋŌcíėė°yķfôîŨĄĄĄøë¯ŋĐēuk¨T*###GŽAAAAû3kÖ,Ŧ_ŋ>>> Á–-[āááN:=đ8???øųųUųúšsįđÁ W¯^011ÁēuëíÛˇ¯q߈šF‰H6Ė#‘r0DĘĀ,’6ĢSkĀ€ .Ōmll,žĻVĢqæĖ™­}Du#­ģ 8ģ¸ ĀČ÷€>ķŲËŸa•ßšsgėŪŊĪ>û,z÷îũû÷cųōå˜5k–,Y‚ņãĮŖI“&¸uëVúsäȑŌŽÄƊ ļ9˛ÚVu,--‘’’‚™3g"??ŽŽŽ˜8q">ųä“:—¨>=hô 5.æ‘H9˜G"e`I›Š„:Œ!=z4`Æ #FŒĀ;īŧS—Ŗû¨Õęō ŸaĀĒž@g; 5¯ `ÃqŽŧ"j$’<‘Ŧ˜G"å`‰”Y$mV/Ŧ_|cƌAķæÍĢÜĮŪŪ%%%õq9ēO“&M¤ –Ö@Ŧ @mĀâQ#ŌČ#Ɇy$Ræ‘H˜EŌfõRĀúāƒÄßA¨TU­&NõÍÁÁAÚ §XZYYĨ?DÔh4ōHD˛a‰”ƒy$Rf‘´YŊ­ĩvíZ´iĶ&&&011A›6m°f͚ú:==@LLŒfŖ•Ué?ųôGĸFUi‰HĖ#‘r0DĘĀ,’6ĢˇX_|ņ&OžŒÎ;Ž=ŠˇŪz øøãëã2TÖÖ@RG`‘ÖS esūę iĶĻXļl† "ißŧy3&OžŒôôôē^ĸAeeeÁÚÚ™™™°*š¤E222`gg'mėŌ8z´ô÷ĸ"Ā ^j•DTJķHD˛`‰”ƒy$Rf‘äT×ÚKŊL!TĢÕ ŅhFqqq}\‚ Ģ˛QVÖÖåŋßŊ[¯×‹ŽŽÆÖ­[¸JĨBvvv•¯¯\š€\\\Äí`ŨēuhÛļ- đõ×_kŋ}ûv´mÛūūūđķķCBBBúîîîņZ?ūøcŽ#ĒŠJķHD˛`‰”ƒy$Rf‘´YŊ°†Ž+Vh´¯^ŊÆ ĢKĐÜŧySŗąb5ŗž×ÁĒIĢ:&L@tt4ĸŖŖņüķĪcúôéâv=Œ­[ˇbčĐĄĮž:u īŋ˙>öíۇķįĪãØącĩZŒpÛļmâĩ^zéĨ:ŊĸûUšG"’ķH¤Ė#‘20‹¤Íz^Ų˙ū÷?ņw•J…5kÖā?ūĀc=8vė1bĈē÷’j¯âŦjĒė{öėÁĖ™3ÅíØØXŦZĩ }ûöÅ°aВ’•J…āā`,X°ŗgĪFVVđØcaåĘ•ØącfΜ [[[ôíÛˇÎŨoßž=@OOŗÆēxņbŧũöÛpvv­œöIDDDDDDD5÷ĐŦS§NIļƒƒƒ—/_ØÛÛÃŪŪįĪŸ¯C÷¨&:uę¤ŲXą¨SMĢoßžbŅiãƍXŧx1 €5kÖĀŨŨüņ€ōųŌķæÍïŋūŠmÛļRSS1nÜ89r­[ˇÆ… ëįUá… đôôDˇnŨ••…įž{sæĖžž~Ž6lJJJĐŠS'|ú駰ˇˇoĐūŌŖĨŌ<‘,˜G"å`‰”Y$möĐŦÔg?¨ĸĸĸ$m|ˆ)„Āœ9spčĐ!XZZâąĮÃ_|ˇß~ŨēuCŸ>}*=îØąc BëÖ­ãĮĮ{īŊ÷PīĨ&ŠŠŠpōäIėŨģ‚ āųįŸĮĒUĢ0qâÄj=tčZ´hĸĸ"ŧ˙ūû9r$öėŲĶ`}ĨGOĨy$"Y0DĘÁ<)ŗHÚŦ^ÖĀ"yi6Öb !œ;wŖGÆĪ?˙,NÍëÜš3ĸŖŖŅŠS'lßž:t€Z­Ö8ļ6˛ėŌĨ ęTųwssÃĀajj 333 0Į×Øīģīžk ´hŅ`hhˆŠS§âđáÃŨĸĘTšG"’ķH¤Ė#‘20‹¤Íē€5aÂ$&&Öhßü›6mzØKQ5lmm5īĀRCá'Ėąy3TRBrr2^xá„……Áßß_l‡……Œe˖!..ŲŲŲ°˛˛Bf…Q];wÆŠS§XŗfM•}=räĸŖŖņpoĀĐĄCņĮ ¤¤jĩūų§¸fVE#FŒk=z4rrrpįÎņõ͛7#00đĄûAT™JķHD˛`‰”ƒy$Rf‘´ŲCO!´ˇˇG›6mĐĨK<˙üķ ŗŗ3LLLpûöm\¸p˙üķļlŲ‚æ͛cõęÕõŲoĒ lĔ„ĩ5vā˙đ&žDŌŽbŗ‹ đå—Ā€åģŽYŗiiixë­ˇÄļyķæáÖ­[X˛d ôõõĄVĢąhŅ"X[[ŖgĪžøüķĪŅž}{tîÜ+WŽÄęÕĢŅ¯_?4iŌƒ Ēķ{Ú¸q#ĻOŸŽÛˇoãįŸÆgŸ}†_~ųxųå— čëëãÉ'ŸÄ¤I“Ē=į͛71pā@¨Õj‚OOO|÷Ũwuî+QE•æ‘ˆdÁ<)ķH¤ Ė"i3•P›ų_÷IMMÅÚĩkąeËœ;wNōšĨĨ%zõę…ņãĮ#44´ÎmHYYY°ļļFffĻV>Ņ.""BcJŪŽOcĐŧļ(ũÃ-h§R•ūsÛ6i‹ˆęGey$"y0DĘÁ<)ŗHrĒkíåĄG`€ƒƒfĖ˜3fāΝ;¸zõ*ōōōĐ´iSxyyAUV-ĄFĨVoŽôŅ(^€ ”ąĻNú÷jøā>""""""""ŲÔŠ€U‘ lllęëtT ^^^’íǁ¤Tã*÷ 1ątŋîŨ¸sD˜ûķHDōa‰”ƒy$Rf‘´ŸB¨rss%Û))5;ŽĻûQÍŨŸG"’ķH¤Ė#‘20‹¤ÍXĀŌ)÷Uĸœœjv\M÷#ĸšģ?D$æ‘H9˜G"e`I›ą€Ĩƒēv\\¨PRéë*āęZē‘ŌÕé)„ēB۟B¨VĢĄßjė;vƒ–°>…¨ŅT–G"’ķH¤Ė#‘20‹$§ēÖ^ęmVqq1ūúë/ŦZĩ wīŪ\ŋ~ŲŲŲõu Ēšsį4Ú ļŲODs$KÚ]\Xŧ"jH•å‘ˆäÁ<)ķH¤ Ė"iŗzy áÕĢWņôĶOãÚĩk(((@īŪŊaii‰… "??+WŽŦËPōķķ+mĐ<ũĶÜqXŋRžûNÎ*tí °āNÔpĒĘ#5>æ‘H9˜G"e`I›ÕËŦ7ß|!!!¸}û6L˙ŸŊ;‹ĒŪ˙8ūpAD6EAqgqAÁ}IËĨ,[ČJ­Ÿiˇë-ëļYVˇÍļ[Ū6­›iuÍļĢ×ë’ļ™KŌ5wEÜwEQD”˜ß#ƒÄ"ËĀœŪĪĮcælŸ™æ=ÚĮīųž:ulĪßqĮŦXąÂ§øøøŊĸAÜÉĨÎ FŪrŽūũÕŧŠlÅæQDĒœō(bĘŖˆ1(‹âĖė2ë÷ßgõęÕÔĒUĢĀķ͛7'))Š˜ŊÄ^š5kVôŠ ō—O'œßKÄŲ›GŠrĘŖˆq("Æ ,Š3ŗËŦÜÜ\rrr =Ÿ˜˜HŊzõėq )Áöíۋ^Ņ°aūrjjÕ#RÍ›GŠrĘŖˆq("Æ ,Š3ŗKkĐ AL™2ÅöģÉd"##ƒW^y…ĄC‡ÚãRWĀRKDDDDDDDœ”].!üāƒ0`ááádff2jÔ(öīßOƒ ˜={ļ=N!% )zÅ/!‘JWlE¤Ę)"ÆĄ<Šƒ˛(ÎĖ. Ŧ&MšĪėŲŗ‰‹‹#77—?ũéOÜ{īŊ&u—Ę‘]ô ]B(RåŠÍŖˆT9åQÄ8”GcPřŲĨP§Nxāxā{RJ)11‘ĻM›^ĄKEĒ\ąy‘*§<Š‡ō(b Ęĸ83ģĖåîî΀HKK+đüɓ'qwwˇĮ)¤ųä<<<ČĖĖ´CU"åWbE¤J)"ÆĄ<Šƒ˛(ÎĖ.#°>LũĢį[Zˇn͖-[8tč=N!%Øēukņ+¯žáéĶĨ>æŒ3pssŖOŸ>œ={–?ū˜V­ZQ§Nüüü4hģwī 66“ÉD׎];v,~~~´k׎õë×0fĖÛč¨ŧĮ˜1c9r$#GŽ,ķkΓ––ÆØąc™8q"5*÷qDėĨÄ<ŠH•REŒCy1eQœ™Ũ/!ŧš‡‡͛7¯ĖSČĩ”ãN„ĶĻMãá‡fȐ!,]ē___7nĖã?ÎÔŠS?~<+WŽäÁ,°ßĻM› &&†}ûöņėŗĪđđÃ3{ölžüōKčÕĢ×5ëČČČ 55•ÔÔTŌĶĶ‹Üæ/ų Mš4aŌ¤IĨzm"""""""â|Ę}=™ŋŋ?ûöíŖAƒøųų•8QZZZyO#ĨĐŦYŗâW^=Ģ Ŧ¤¤$yä† ÂâŋŠYŗ&§OŸæíˇß&99Ųļm\\\}ÃÃÙÜöČģdpđāÁ4nܸÂĮc0Y,Kyw;v,5kÖäĶO?āÜšsDDD™™I`` ģvíbŅĸE :ÔnWŗŲŒéééx{{;ēœ2ËĖĖÄÃÃŖđŠc `՝`Á: ËæĘ/}įApLT(R}›GŠrĘŖˆq("Æ ,Š#U´÷RĄXĢW¯føđáļßŋúę+˛ŗŗŲŋ?[ˇn劧žâwŪŠČ)¤ .,ôüwß}Gũúõíq Š¨æÍķ—q\""""""""ed—ÖĢ¯žĘsĪ=ĮÍ7ßĖoŧÁoŧÁ-ˇÜÂķĪ?ĪĢ¯žZæã%%%qß}÷Qŋ~}<==‰ŒŒdķæÍ%îķÛoŋ…‡‡-[ļdúôéå}9N'00đÚ]ŨĀJH¨´ZDĒģRåQDĒ„ō(bĘŖˆ1(‹âĖĘ=‰ûÕƌCXX~ø! ,ĀbąÎęÕĢéŪŊ{™ŽuæĖz÷î̀øųįŸ āāÁƒøúúģĪáÇ:t(ūķŸųæ›oXŊz5ãĮ§aÆÜyį|uÆįééyíBBō—ÕĀŠ4ĨĘŖˆT åQÄ8”GcPřŲĨĐŊ{wžũöÛ gōäÉķÅ_؞ šēųR„éĶ§ĶŦY3۝ÃÂÂØ´iīžûnĩh`ûŦÄ}ÖŽ]ËāÁƒ <7dČ6mÚÄå˗ mŸ••…Ųl.đpyuę@ŖFÖéåt˛ŗŗK\?eĘ[îūčÖ[o-uķjŅĸE|ûíˇŦ[ˇŽ;v°|ųrjÖŦYæzEDDDDDDėÉ.#°~ûí7nēé&z÷îÍ˙ū÷?Ūxã ØļmŸū9ķæÍ+õą:Ä'Ÿ|ÂSO=Åßūö76lØĀc=FíÚĩ=zt‘ûœ8q‚Fy͙+5jDvv6ŠŠŠ…Žķ}ë­ˇŠœ›kĶĻMÔ­[—.]ē°{÷n.^ŧHŊzõhŅĸÛļm yķæäæærėʝü"##9pāÔ­[—ļmÛ˛eË‚‚‚pwwįȕ†QĮŽIHHĀl6ãááADD„mn¯&MšāááÁĄC‡hßž=‰‰‰œ={–ZĩjɆ hܸ1^^^8pĀVĶūũûIKKŖFDEEąaÃ, 6ÄĪĪ}ûöŅ ^'Ob9~œĢVaōđ k׎lÚ´‰œœęׯO@@ģwī M›6˜ÍfNž< XGŲÅÅÅqųōeüüühŌ¤‰­AŲĒU+.\¸@rr2ŅŅŅėØąƒĖĖL|||hÖŦÛˇoŦ#ę˛ŗŗILL K—.ėŲŗ‡ .āååEĢV­Øēu+͚5āčŅŖtęԉƒ’‘‘§§'ĄĄĄÄÅÅŲŪī5jpe„Y‡8zô(éééxxxĐž}{Û]7ņôôäāÁƒDDDpüøqΜ9C͚5éŌĨ ëׯˇ}–ŧŊŊŲŋ?`ᗒ’ÂéĶ§qww'::š7’››KÆ ņ÷÷gīŪŊ´mۖ3gÎpęÔ)L&ŨēucķæÍdggãīīĀO<Á† 8ū<?ū¸í˛Û=zđä“OKįΝyá…xāHLL¤f͚Ü˙ũ 0€ũë_?~œaÆQŗfMŪ~ûm._žLrr2ëׯgŨēu,]ē”—_~€Î;ŗoß>Ο?——­[ˇ&>>€7ß|“§Ÿ~š={öØ>ŗ{÷îåÜšsÔŠS‡°°0ÛûŨ´iSjÕĒÅáÇmī÷ącĮ8{ö,ĩk×ĻcĮŽlܸŅö™­[ˇŽíũįĉ¤ĨĨzŋđņņąŊߥĄĄ¤ĻĻ’ššŠ››]ģvĩŊß 4 AƒļzÛ´iCzzē­™wõgÖßߟƍŗk×.Ûgöüųķœ8q€Ž]ģ˛mÛ6˛˛˛đõõ%88Øö™mŅĸ—.]"))Éö™-î;" €ãĮę;",,Œ“'O^ķ; ]ģv¤ĨĨqęÔ)Ûû­īĮ}G4jÔČö~ˇnŨšŒŒ Ûgļ[ˇnÄĮĮséŌ%|}} bĮŽ´lŲ’ĖĖLŽ?@TT;wî$33oooBBB üš–““c{ŋKúŽÆÍÍ­ĀgöđáÆũŽ¨W¯ééé†úŽ0âß#ôĄīˆĒøŽ¨QŖ—/_6Ôw„˙Ąī}GTöwDvv6ëׯ7Üw„˙ĄīûGTt6“ÅbąTč@Īž=šëŽģxꊧ¨W¯[ˇnĨe˖lܸ‘ÛoŋŨöĸ4jÕĒEtt4kÖŦą=÷ØcąqãFÖŽ][ä>mÛļeėØą<˙üķļįV¯^MŸ>}HNNĻqãÆļĪĘĘ"++ËöģŲl&88˜ôôtŧŊŊK]ĢQėÛˇļmÛ^{Ã#ā?˙ą.īß­[WnarM&“‰I“&ņĘ+¯pčĐ!ēuëƖ-[Æd2ņæ›oōˇŋũ €{–-[ōÖ[o‘’’BTTķįĪ§[ˇn„„„đÃ?Đž}{ ā%„ŗfÍâ‡~(U#Ųßߟ‰'ōũ÷ߓ••ÅčŅŖyėąĮ*õ=p5ĨÎŖˆT:åQÄ8”GcPőĖf3>>>åîŊØåÂíÛˇsĮwzžaƜ>}ēLĮ $<<ŧĀsaaaļŽxQ7nlëlæIIIĄFÔ¯_ŋĐöĩk×ÆÛÛģĀٝ9sĻtj"wCzđÁëŋtôéĶ‡UĢVŲÖ=đĀļåå˗ķČ#Ö9ˆ‰‰aŊv­%;;›ƒōŋ˙ũĨK—ōŲgŸņĶO?ŲõŽŽÔy‘J§<Š‡ō(b Ęĸ83ģ4°|}}mÃ9¯ļeËš6mZĻcõîŨÛ6$2Īž}ûh~õ$äĐŗgO–-[VāšĨK—]-æī)õkŦ¤V||úč#‚ƒƒ‹ÜîÃ?dÛļmtėؑđ /Đ­[7Ā:OÜØąc‰ŒŒ´M’XO>ų$'Nœ <<œ¨¨(nēéĻ"/–â•:"Ré”GãPEŒAYgf—IÜķ:tˆ¸¸8rsséÜš3mÚ´ą×Ą+UE's´õë×Ûî\Wĸ‹!¯ąĶģ7üūû5wųæ›oxīŊ÷øß˙ūĮįŸÎîŨģmŖ€ŌŌŌđ÷÷/4IxJJ aaaŦYŗ†víÚņüƒgŸ}–sįÎē$Ž(cƌ!::šG}´TëētéÂ-ˇÜÂoŋũ†Ųlæ–[naŌ¤I¸ģģ—xžôôtZˇnÍ÷ßO=X¸p!111ĖŸ?Ÿ˜˜˜kÖY” `øpøcĒōŽô›7ĘyhqĨÎŖˆT:åQÄ8”GcPő 1‰{ž–-[2|øpîŧķNΟ?¯ âĒHŖFJˇa:ˇm).!\šr%“&MâĮ¤^ŊzôčŅƒ%K–0aÂ/^LŨēu‹ÜoŨēutéŌ…víÚ0nܸŌÕWN—/_fķæÍ,Y˛„ÕĢWŗvíZfĖ˜qÍũ|||˜?>Ī=÷QQQÄÆÆ^îëÂsrāņĮ 7¯ ˙š'ž(ųrBq~ĨÎŖˆT:åQÄ8”GcPřŲĨõÄOđ¯ũ €œœŽģî:ētéBpp0ąąąö8…” ,Ëœæ-‰å:f'õ#véĨb›);vė`ėØą,Z´Čvi^Īž=‰§{÷îĖŸ?ŸŽ]ģ’SÄĘ2¨¯W¯^DFFVč_š7oΝwŪI:uđôô$&&ĻȉĮŋúę+Ûe‘_|ņũúõ#66–Í›7ķüƒãĮÛ.),ĢUĢ 1ąøõ ;fŨN\—3ŽâqUĘŖˆq("Æ ,Š3ŗKkŪŧyļ‰žŋ˙ū{:Ğ={xâ‰'xá…ėq )ÁūũûKĩŨĸo3ø î:Zqģų­†´â˙×YômÁģ&%%qûíˇķÅ_a{ūđáÃxyyq÷ŨwķŅGąoß>222 M ŪŗgOļlŲž}ûøüķĪ‹­i͚5ÄĮĮWčZėQŖFątéRrssÉÉÉaŲ˛eEN<>zôhÛ¤åcĮŽ(p÷Ė×_ë¯ŋžÖ­[—ĢŽ"nÄYĄíÄ9•6"Rų”GãPEŒAYgf—Vjj*7Ŧ€ß}÷Ũ´mۖ?ũéOlßžŨ§ ZômA÷]Į“Ų˙ ˜DÜÉ%˜Dž4O"čžë 4ą>˙üsN:œO>ią´xņbbcc‰ŠŠ"22’ŪŊ{ķÎ;īāããà 7ÜĀųķįéÔŠ=ô|úé§ 6Œ^Ŋz9ņzY}ķÍ7ņß˙ū——^z‰   ļlŲĀˆ#hŌ¤ tęԉĀĀĀ"įĪ*ĘK/ŊDhh(­[ˇæȑ#ļ‘„åhßíDDDDDDDÄĘ.“¸7oŪœĪ>ûŒn¸-Z0mÚ4nšåvîÜIŸ>} ?–ŗOân6›KŦ;'>đ'͓p'ˇđzÜøĀ{OĻŊÄ5æ=—ääXī6˜”Tô˙üsƍĮęÕĢŠ]ģ6îîî<÷Üsö8…”āôéĶ%ŽON†&/vŊ ëzÍÍTq110o4mZđų  ëķ11ŽŠKĒÎĩō("UGy1åQÄ”Eqf5ėu áÇøũėŲŗÜ˙ũö:ŧ”ĀũCzá8MĻč[äY°Ž×ÜLöˇŨfų–œl}˙ûöÕČĢęâZy‘ĒŖ<Š‡ō(b Ęĸ83ģŒĀšJxx8˖-ÃËË:ĸ'99™ņãĮÛãR‚Ŋ{÷–m‡üå„{–"Rí•9"Ri”GãPEŒAYgf—X5kÖ,r˛ö'žx‡{SKDDDDDDDœˆ]F`|ũõ×ôéĶ‡&MšpäČĻL™ÂĸE‹ėu )FÛļmËļCŗfųËWū[‰ˆ}”9"Ri”GãPEŒAYgf—Ö'Ÿ|ÂSO=ÅM7ŨÄŲŗgmˇûúú2eĘ{œBJpæĖ™˛íPˇ.äMŪ§X"vUæ<ŠHĨQEŒCy1eQœ™]X}ôŸ}ö/ŧđîîîļįŖŖŖŲž}ģ=N!%8uęTŲwĘģŒ0) .]˛k="ÕYšō("•By1åQÄ”Eqfvi`>|˜Î;zžvíڜ?Ū§˜LĻ˛ī”×ĀĘͅÄDģÖ#R•+"R)”GãPEŒAYgf—V‹-ˆ/ôüĪ?˙Lxx¸=N!%čÖ­[ŲwrÁ‰Üããã™;wn‰Û˜L&222Š]?}út"##‰ŒŒÄßߟ   Ûī+WŽdæĖ™tčЁ5jđĪūŗĐūķįĪ§C‡DDDNB)ßÛĮ{ŒL&;vė(°nėØątėؑČČHēvíʊ+JuLqŒråQD*…ō(bĘŖˆ1(‹âĖėrÂgžy†Gy„ĖĖL, 6l`öėŲŧõÖ[|ūųįö8…”`ķæÍDEE•m'm`ũđÃÜ}÷Ũå>ÆC=ÄC=Ā˜1cˆŽŽæŅGĩ­÷÷÷gîÜšŧõÖ[…öŨ˛e /žø"+VŦ I“&˜ÍfjÔ(]ĆÎĉéĶ§OĄu|đžžž€õ58S§Né_O Ē\y‘JĄ<Š‡ō(b Ęĸ83ģ4°ÆŽKvv6'Nä… Œ5ŠĻM›2uęTFŒaSH ˛ŗŗËž“4°~úé'ūöˇŋŲ~ßŗg3fĖ`čĐĄÜ{īŊ$''c2™ˆŠŠbōäÉŧüō˘Íf"##éŅŖĶ§OgÁ‚üíoÃĪĪĄC‡V¸ĻN:āæVxđâ{īŊĮ„ hŌ¤ ŪŪŪĨ>nŋ~ũŠ]—×ŧ8{öŦWWŽ<ŠHĨPEŒCy1eQœY…XŲŲŲ|ûíˇ 6Œ?˙ųĪ¤ĻĻ’››K@@€=ę“Rđ÷÷/ûNNĐĀ:t¨­éôÍ7ßđŪ{īÃįŸNHHK—. -- ^{í5~øáæ͛@JJ ūķŸYŗf íÚĩã˙øGĨÖģk×.ZļlÉu×]‡Ųlæ–[naŌ¤InlP^Ī=÷˙ũī9sæ ,PËĀʕGŠĘŖˆq("Æ ,Š3ĢđX5jÔāá‡&++ € ¨yUÅ5jTöš7Ī_6h+Īʕ+™4i?ūø#õęÕŖG,Y˛„ &°xņbęÖ­[ä~ëÖ­ŖK—.´k×€qãÆUj—/_fķæÍ,Y˛„ÕĢWŗvíZfĖ˜a—cŋũöÛÚļm[â ÖŦYSá×4jÔ(/^Ė˜1c°X,,[ļŦČš­FÍčŅŖKuĖėėl>L›6mØ°a)))´lŲ˛ÂõŠˆˆˆˆˆˆ8 ģ4°îšį{ė1Ûs&“ ‹Å‚Éd*rtŒØOë֭ˡcHlÜšš˜UÔY°ÜzĘNœ8aSH .]ēTžŊŧ Aër4°úöĩŪm0oÂö?2™ 8Øēˆŗ*wEÄî”GãPEŒAYgf—V˙ūũYĩjUĄį˙ũwúĒQé|}}ËŋsŪe„‰‰pų˛=Ę)–ģ;Lj]ūc+ī÷)S4ģ8ˇ åQDėJy1åQÄ”Eqfv™ÄũÖ[oåŲgŸeķæÍôčŅ€uëÖņß˙ū—W_}•Å‹ØVė+((¨ü;‡„ĀĻM›kmbĩhaˇēŠķæYīFxõ„îAAÖæULLĨž^¤ŌU("bWĘŖˆq("Æ ,Š33Y,KEâæVē\&“‰œœœŠžÎîĖf3>>>¤§§ãíííčrĘlũúõtīŪŊ|;?ķ ŧûŽuų×_aĀûV‚œëŨ““­s^õíĢ‘Wâ*”Gą+åQÄ8”GcPő*Ú{ąËŦÜÜ\{FĄŠīD˜ĮŨú÷¯˛Ķ‰ˆˆˆˆˆˆˆŗËXâX-[ļ,˙Îj`‰¸Ē åQDėJy1åQÄ”Eqfv°aÃbccIII)4"ëũ÷ߡ×i¤™™™åßųę֑#ŽE¤ēĢPEÄŽ”GãPEŒAYgf—Öß˙ūw^|ņEÚĩkGŖF0]u‹9Ķo7'vwüøq‚ƒƒËˇsķæųˁ%RaĘŖˆØ•ō(bĘŖˆ1(‹âĖėr áÔŠS™9s&ģwī&66–•+WÚŋūúĢ=N!•ÅË 4°.>ėØZDDD¤Ō˜L&222*|œ)SĻ’’bû}Ō¤I<ũôĶ,^ŧ˜gžyĻTĮŲŋ?ƒ ĸS§NDDDđŸ˙ü§Âĩ‰ˆˆˆë˛KËÍ͍ŪŊ{ÛãPRQQQ;@ŪuĐĮŽ†”ŠTH…ķ("vŖ<–Ovvv‰ë˙ØĀēÚ­ˇŪĘ;īŧSĒķŒ3†{īŊ—­[ˇōë¯ŋōĖ3Ī””TæzÅ9("Æ ,Š3ŗKëÉ'Ÿäã?ļĮĄ¤vîÜYą´nmũiąh–HU8"b7Ęcaīžû.Ŋ{÷ĻmÛļĖž=ÛöŧÉdâŊ÷ŪŖ˙ū<˙üķœũôS^{í5Ž?ÎđáɌŒ$>>žĀņg͚ÅđáÃKUËÖ­[:t(5ĸS§N…å”GcPřŲeŦ§Ÿ~š›ož™V­ZN͚5 Ŧ_°`=N#ŨđD|mÚä/8aa;žH5Ļ‰1EŒCy,Ėd2ązõj:DˇnŨčĶ§m.”ŦŦ,bcc¸įž{ eá…¤¤¤Edd$/ŋü23gÎdŪŧy´o߀īžûŽ\ĩtíڕožų†§žzŠƒ˛fÍZ´ha—)¤<Šƒ˛(ÎĖ.#°ūú×ŋ˛råJÚļmKũúõņņņ)đĘåíí]ąäĀØŋŋbĮŠæ*œGąåą°|°ŪFŊOŸ>ŦZĩĘļ-/_žœGy€€€bbbXąb…]k™5kk׎%22’gžy†úGPqĘŖˆ1(‹âĖė2ëĢ¯žbūüųÜ|ķÍö8œ”QHHHÅpuëĀŠK¤šĢpEÄn”ĮkģúnŅ^^^ÅŽ+ę÷˛ØĩkŖF wīŪ|üņĮ4oŪœ˙ū÷ŋļmnŧņF\îsˆą)"Æ ,Š3ŗË,ZĩjeCI9lÛļ­b¸úBĀŠ įQDėĻēä1'bcaölëĪœœâˇ9s& üūûīôéĶ§Číh›÷ęÔŠS,\¸ë¯ŋ°ūë}zzz™j '>>žøøxÛŧŠ'OžÄbąđË/ŋhr‰ëŠ.y1:eQœ™]X“&Mâ•W^á… ö8œT5đõĩ.k–ˆˆˆĶX°BB`Ā5Ęú3$Äú|Qj׎MīŪŊʼnT3vÉŖˆØ…ĢįqÁ>ÜzáĢå]é7oÄÄT}]"Eqõ<Š8 eQŠĸŊģĖuûíˇÛã0RN9%]+PZmÚXXššpø0„†Vü˜"Ր]ō("váĘyĖɁĮ/Üŧës&<ņÜv¸ģWyy"…¸rEœ‰˛(ÎĖ. ŦW^yŇ‘rJLL¤iĶĻ;Č'rWK¤\ė’Gą WÎãĒU˜Xüz‹Žŗn×ŋ••%R,WÎŖˆ3QřŲĨ•gķæÍėŪŊ“ÉDxx8;wļįáĨ2i"w§‘œlßíDDDDŒÎ. Ŧ””FŒAll,žžžX,ŌĶĶ0`sæĖĄaÆö8Ã.ÂĢG`Š%Rnj܋‡+į10ĐžÛ‰T6WÎŖˆ3QřŲå.„ũë_1›ÍėÜš“´´4Μ9Î;0›Í<öØcö8…”`ßž}?HÛļųË{÷Vüx"Ք]ō("váĘyėÛ‚‚ō'l˙#“ ‚ƒ­Û‰+įQę(‹âĖėŌĀZ˛d Ÿ|ō aaaļįÂÃÃųøãųųįŸíq )Áųķį+~úõĄAë˛X"åf—<Šˆ]¸rŨŨaęTëō›XyŋO™ĸ ÜÅ8\9"ÎDYgf—Vnn.5kÖ,ô|͚5ÉÍÍĩĮ)¤^^^ö9Pģv֟IIpîœ}Ž)RÍØ-"RaŽžĮ˜˜7ū8oPõų˜ĮÔ%RWĪŖˆŗPř™,–ĸnĀ\6ˇŨvgĪžeöėŲ4iŌ€¤¤$îŊ÷^üüüX¸pa… ­LfŗŌĶĶņöövt9e–••EíÚĩ+~ ?ũ fδ.oÚQQ?ĻH5cˇ<ŠH…U—<æäXī6˜œlķĒo_ŧãŠ.y1:eQŠĸŊģŒĀúį?˙Éšsį ĄUĢV´nŨš-ZpîÜ9>úč#{œBJoŸåĀ]F(RNvËŖˆTXuÉŖģ;ôī#GZĒy%FT]ō(btĘĸ83ģ܅088˜¸¸8–-[ƞ={°X,„‡‡3pā@{^ĒŠX"""""""b@vi`å4hƒ ˛į!Ĩ‚ƒƒís 5°D*Ėny‘ SEŒCy1eQœY….!üõ×_ Įl6Z—žžNDDĢV­ĒČ)¤ÜÜėr%(´l™ŨX"åbˇ<ŠH…)"ÆĄ<Šƒ˛(ÎŦBŸŪ)SĻđį?˙šČɡ|||øË_ūÂûīŋ_‘SH)9rÄ>ĒUËÚÄØˇtI‘2ŗ[E¤Â”GãPEŒAYgVĄÖÖ­[šņÆ‹]?xđ`6oŪ\‘SHUËģŒđÂHJrl-""""""""T°uōäIjÖŦYėú5jpęÔŠŠœBJĄcĮŽö;XhhūōîŨö;ŽH5a×<ŠH…("ÆĄ<Šƒ˛(ÎŦB ŦĻM›˛}ûöb×oÛļĀĀĀŠœBJáđáÃö;Xxxū˛X"ef×<ŠH…("ÆĄ<Šƒ˛(ÎŦB ŦĄC‡ōōË/“™™YhŨŋyå•W¸å–[*r )…sįÎŲī`W7°ví˛ßqEĒ ģæQD*Dy1åQÄ”Eqf5*˛ķ‹/žČ‚ hÛļ->ú(íÚĩÃd2ą{÷n>ūøcrrrxá…ėUĢŖN:ö;XXXū˛X"ef×<ŠH…("ÆĄ<Šƒ˛(ÎĖdąX,9Ā‘#Gxøá‡ųå—_Č;”ÉdbȐ!L›6{ÔYŠĖf3>>>¤§§yGEŖģ|ųr‰s‘•Yp0$&‚Ÿœ> &“ũŽ-ââėžG)7åQÄ8”GcPő*Ú{ŠĐ%„Í›7į§Ÿ~"55•õë×ŗnŨ:RSSų駟œĸyå âââė{ĀŧËĪœ”û[ÄÅŲ="RnĘŖˆq("Æ ,Š3ĢĐ%„WķķķŖk׎ö:œ8Rx8,]j]Ūĩ 5rl="""""""R­Ux–8^ĶĻMí{@Mä.RnvĪŖˆ”›ō(bĘŖˆ1(‹âĖÔĀrĩjÕ˛īÕĀ)7ģįQDĘMy1åQÄ”Eqfj`š€Ã‡Û÷€ēĄHšŲ="RnĘŖˆq("Æ ,Š33\kŌ¤I˜LĻÆģ}lllĄíM&{öėŠÂĒ]Œŋ?äŊį;v@ÅnT)"bH&“‰ŒŒŒRoüøq pÍí233šũöÛiÛļ-‘‘‘Üxã$$$”ęššš<ũôĶ´oߞĐĐPūô§?qéŌĨR×(""""âĒ ×Āˆˆˆ 99ŲöØž}û5÷Ųģwo}Ú´iS•C‡ėĐöí­?SSáäIû_ÄEUJÅš4iÂʕ+Kĩí¸qãØģw/ņņņÜrË-Œ7ŽTûũë_˙bÛļmÄÅÅą{÷nĻNZîšĢ;åQÄ8”GcPř˛UŖF 7nl{4lØđšûØĮŨŨŊ *5†cĮŽŲ˙ ;æ/oÛf˙㋸¨JÉŖTšwß}—ŪŊ{Ķļm[fĪž ĀO?ũDdd¤íáááÁ—_~IBB 4°ík2™˜4iRĄũ=<<:t(&“ €=zpčĐĄRÕŗuëVH­Zĩ0™L :”¯ŋūÚίēúPEŒCy1eQœ™!Xû÷ī§I“&´hŅ‚#F”ę/ū;w&00n¸áš˙Bž••…Ųl.đpfgĪžĩ˙A¯îĖ—bœˆXUJĨŌ˜L&V¯^͒%Køë_˙ĘącĮ:t(ņņņÄĮĮķôĶOFLLLŠ÷˙Ŗ?üaƕĒžŽ]ģ˛hŅ"Ν;ĮĨK—˜3gNŠ/?””GãPEŒAYgVÃŅüQ÷îŨųęĢ¯hÛļ-'Ožä7Ū W¯^ėÜš“úõëÚ>00O?ũ”¨¨(˛˛˛øú믚ᆈĨ_ŋ~Ežã­ˇŪâÕW_-ôüĻM›¨[ˇ.]ēta÷îŨ\ŧx‘zõęŅĸE ļ]…Ôŧysrssm˙“ɁČČČ nŨē´mۖ-[ļ„ģģ;GŽ cĮŽ$$$`6›ņđđ ""‚Í›7ÖKS<<ūøcV­Ze÷×_("ÆĄ<Šƒ˛(Žd6›ņņņ)wīÅđŸÜēuëŌĄC[‡´4zôčQâöĩk×ÆÛÛģĀÙåužíĒNțįNČÎļ˙9D\PĨäQĘ$'bcaölëĪœœâˇ9s&`mNũūûīôéĶ‡¤¤$nŋũvžøâ [ķĒ,ûŧ˙ūûĖž=›e˖Ûŧ˜7ožírÅúõ듙™iڟššĘÛoŋÍĉKûŌå”GãPEŒAYgf¸K˙(++‹ŨģwĶˇoßRīŗe˖ MŦķ`íŨ YYpā„†:ē"‘-X?WFąS§BQĶXÕŽ]›ŪŊ{sęÔ)>úč#‚ƒƒyõÕW9uęO>ų¤mģ×^{ŽWßÜĸ„ũ™0a-[ļdĀ€ļíō†Ž—$==ëŽģwwwrrrxâ‰'J=–ˆˆˆˆˆ+3Ü%„O?ũ4Æ ŖYŗf¤¤¤đÆođÛoŋą}ûvš7oÎķĪ?ORR_}õSĻL!$$„ˆˆ.]ēÄ7ß|ÃÛoŋÍüųķ‹t÷*:ŒÍŅŽ9BķæÍíā×_‡—_ļ.Ī wŨe˙sˆ¸˜JËŖ\Ķ‚0|8üņOĩ+7dŪŧĸ›Xåe2™8wî^^^ö;¨Ø•ō(bĘŖˆ1(‹âHíŊnVbb"#GŽ$55•† ŌŖGÖ­[g Yrr˛m’=€K—.ņôĶO“””D:uˆˆˆāĮdčĐĄŽz UŽnŨē•sāĢīD¸uĢX"ĨPiy”åäXG^õO2‹ĩ‰õÄpÛmāî^å剃("ÆĄ<Šƒ˛(ÎĖp Ŧ9s攸~ÖŦY~Ÿ8qbĩŸäāÁƒ4hĐĀūîÔ)ųʝ#D¤d•–G)ŅĒU/ü#‹Žŗn×ŋŋ}Îi°ĖRåQÄ8”GcPř~wq ț|øĘ­8EDŒčĘ]ĩíļˆˆˆˆˆ‹X. <<ŧrl2AįÎÖåãĮ!%ĨrÎ#âB*-RĸŌŪˇC÷÷¨^”GãPEŒAYgĻ– 8qâDå<¯×…•€Éd"$$¤ōę)‚Éd”7Ks)ôīß“ÉDlllß¯~L™2ĨrŠ—WŠy”bõíkŊÛ`q_&[ˇ“ęCy1åQÄ”Eqf†›KĘ.--­ō~u+.† Šŧs•ĶėŲŗ+|Œ đŅGŲ~īŌĨK…)ÕSĨæQŠåîS§ZīBh2œĖ=¯Š5eŠ&p¯n”GãPEŒAYgĻX. f͚•wđ2ŒĀúŖ3fāææFŸ>}8{ö,ü1­ZĩĸN:øųų1hĐ vīŪ @ll,&“‰Ž]ģ2vėXüüüh׎ëׯ`Ė˜1…FI3€‘#G2räČ ŊĖēuërË-ˇp×]w1bÄÚļm[ĄãIõUŠy”ÅÄĀŧyĐ´iÁįƒ‚ŦĪĮÄ8Ļ.qåQÄ8”GcPřŠå*u´Pģvāáa].CkÚ´i<üđà 2„ĨK—âëëKãƍyüņĮ™:u*ãĮgåʕ<øāƒöÛ´iÄÄÄ°oß>ž}öY~øafĪžÍ—_~I@@Ŋzõēf¤ĻĻ’ššJzzząÛ=z”zõęáááÁ 7ÜĀJũZEŽĻŅ{Ž °r%üûß֟‡ĢyU])"ÆĄ<Šƒ˛(ÎL ,7JŠRÔ¨;Z—ŗųšģ$%%ņČ#0xđ`/^Œ§§'§OŸæíˇßæ/ų ˙ûßÉÉÉ!..ŽĀžáááLž<™įž{îĘ)­¤îŨģs×]wņũ÷ߓ’’¤I“7nÜ5kyôŅGiØ°! 6äļÛn+r›;Yŗfņũ÷ßsĪ=÷đë¯ŋÚFw‰”UĨæQJÅŨú÷‡‘#­?uŲ`õĨ<Š‡ō(b Ęĸ83́%×ÖĨ lÜĄĀīī@Ô Đ°/¸ũ…uęÔÁŨŨuëÖąuëVĸŖŖšpáãĮĮÍ͍™3gÄ°aÃČĖĖ,°oÆ üĄ­ŲŲŲX,|đAæ͛Į„ xå•WJUúĉšīžûđķķ+r›ĮÜļÜŗgOžũö[ļmÛVĒ㋈ˆˆˆˆˆHåSËä]RWi:åĀMĀ:ā–7Āī â /OƒĐ{ mîīīĪėŲŗ4hƒfųōå´kדÉDvv6iiiŦYŗ†ŦŦŦR—đ /0kÖ,BCC‰ŽŽfΜ9´hŅ‚îŨģ—¸_xxx‰ˇŠMOOįĻ›nâöÛo' €ųķįЧOŸR×&rĩJĪŖˆ”šō(bĘŖˆ1(‹âĖt Ą đņņŠŧƒīų&?i€åĘĪ9f¸é>ëú"ôėŲ“Å‹sņâE Äūũû™6m 4āũ÷ß'((ˆúõë—ēŒ5kÖXËŲŗĮ6iû'Ÿ|Rá—Wģvm‚ƒƒųđÃųË_ūÂæ͛;v,_~ųe…-ÕSĨæQDĘDy1åQÄ”Eqf&‹åꛍWOfŗŌĶĶņöövt9eļ~ũúkŽD*—ܸĪßÚŦ*ęSbFxÃ7iÅ^N(RŨTZE¤Ė”GãPEŒAYGĒhīE#°¤x§VÁ/Å4¯Āúü/fëv""""""""•D ,Z9ž˜ gŽąÍ™+ۉP‰y‘2SEŒCy1eQœ™X. 55ĩr\'Šžq_>ŋ+ۉP‰y‘2SEŒCy1eQœ™X. Ōž„öĩŪmĐTĖzÖõ ûVÎųEœūR bĘŖˆq("Æ ,Š3S˸šUŌF7wxy4§p˄õų—§iw‘ĢTZE¤Ė”GãPEŒAYgĻģâüw!Ŧt{ž…×Æ['l?ƒõ˛Á^đętŊ×ŅՉˆˆˆˆˆˆˆÁé.„Âƍ+÷Ą÷Â7iđ⟡ĐúĄšW"E¨ô<ŠHŠ)"ÆĄ<Šƒ˛(ÎL ,››[ų'qs‡N#`-°Ø°ŠōĪ)â„Ē$"R*ĘŖˆq("Æ ,Š3SË4hĐ jN Ļ+“a­__5įq2U–ĮJĪÜšsKÜÆd2‘‘‘QėúéĶ§Idd$ūūūŲ~_šr%3gΤC‡Ô¨QƒūķŸ…öŸ?>:t ""‚đđpJUûc=FHH&“‰;vXˇqãFz÷îMĮŽ‰ŒŒä×_-Õ1Å9šJE\ō(b Ęĸ8ŗŽ.@*ŽĘž„ŧŊ!, v킭[!3<<ĒæÜ"NÂUūRĪ?üĀŨwß]îc<ôĐC<ôĐCŒ3†ččh}ôQÛzæΝË[oŊUhß-[ļđâ‹/˛bÅ š4i‚ŲlĻFŌũ‘5|øp&NœHŸ>} }š´´4 @hh(žžžüüķĪxbdFÍŖHu¤<Šƒ˛(ÎL ,)›îŨķ—uĄˆĶ:t(ņņņÄĮĮķôĶOFLL ß|ķ !!!lßžmÛļņŪ{īĀk¯ŊÆĀ‰gúô餤¤đį?˙™E‹ąvíZj׎]ŠõîÚĩ‹ŖGrŨu×Ņšsg^zé%rrr*tĖ ШQ#æĪŸĀúõëŲˇo_Š/MĮQË´iĶĻęNvuk͚Ē;¯ˆ“¨Ō<–Ãʕ+™4i?ūø#õęÕŖG,Y˛„ &°xņbęÖ­[ä~ëÖ­ŖK—.´k×€qãÆUj—/_fķæÍ,Y˛„ÕĢWŗvíZfĖ˜Qáã.Z´ˆĪ?˙œ.]ē0mÚ4úôéC͚5íPą‘Ņķ(R("Æ ,Š3Sˤ§§WŨÉ:t//ëōīŋƒÅRuįqUšĮ+rr 6fĪļū,n ŌŽ;;v,‹-˛]š×ŗgOâããéŪŊ;ķįĪ§k׎EŽt˛”!ëŊzõ"22’îW7ŧ˨yķæÜyįÔŠSOOObbbØ°aCĄížúę+Ûe‘_|ņÅ5ÛącG~ūųgâââøōË/9~ü8áááåŽSŒÍy‘ĸ)"Æ ,Š3Sˤ¤¤TŨÉjԀ=ŦËĮÃ‘#Uwn'PĨy,€0F˛ū ą>ĩ¤¤$nŋũvžøâ """lĪ>|///îžûn>úč#öíÛGFFŪŪŪū‚ĶŗgOļlŲž}ûøüķĪ‹­i͚5ÄĮĮŗž—5ŠĨK—’››KNN˖-ŖS§N…ļ=z´íŌČącĮ^ķ¸'Nœ°-öŲgÔ­[—ë¯ŋžÜuŠąUuE¤xĘŖˆ1(‹âĖÔĀ’˛ëŨ;yõjĮÕ!RÍ-XÇCbbÁį“’ŦĪ_ŨÄúüķĪ9uęO>ų¤mÄŌâŋ‰%**ŠČČHz÷îÍ;īŧƒ7ÜpįĪŸ§S§N<ôĐCđ駟2lØ0zõęUäÄëeõÍ7ßÄ˙û_^zé%‚‚‚Ø˛e #FŒ I“&DDDĐŠS' ÜÁ°$<ōAAA$&&2pā@Zˇnm[7cÆ ÚļmK›6møūûīY¸p!&“ŠÂ¯EDDDDD*—ÉR–ëB\”ŲlÆĮĮ‡ôôtŧŊŊ]Žņ-_ƒY—~ĻMsl="ÕPNŽu¤Õ›WyL& ‚ÇÁŨŊJK)¤ĸŊĀrqqqU{ÂîŨ!oô…F`‰PUy\ĩĒøæX§§;vĖēHuUå>ŠHą”GcPřŠå._ž\ĩ'ŦWōæĸŲž4 ˆMUå19ŲžÛ‰¸ĸ*˙ķQDŠĨ<Šƒ˛(ÎL ,āīī_õ'͛ËbuëĒūü"UUy ´īv"ŽČ!>ŠH‘”GcPřŠå7n\õ'Ŋz"÷߯úķ‹TUåąo_ëWÅÍ?n2Ap°u;‘ęĘ!>ŠH‘”GcPřŠåvíÚUõ'՝EŠTUytw‡ŠS­Ëlbåũ>eŠ&p—ęÍ!>ŠH‘”GcPřŠ%å ͚Y—ׯ]K-Råbb`Ū‹Z´€#G V-HKƒēu]•H•0\EĒ1åQÄ8”GcPő*Ú{Ņ,pāĀG—PÉ”? ëŌ%ˆuh9"UÉpyŠÆ”GãPEŒAYgĻ– ČČČpt …]}á÷ß;Ž‘*fČ<ŠTSĘŖˆq("Æ ,Š3SËÔ5âåyƒ‡‡uyŅ"ĐŨ.¤š0dEĒ)åQÄ8”GcPřŠåÚļmëč Ģ[ļ.Ÿ8ë×;ļ‘*bČ<ŠTSĘŖˆq("Æ ,Š3SËlŲ˛ÅŅ%íŽ;ō—.t\"UČ°yŠ†”GãPEŒAYgĻ–TžaÃĀíĘGláBëŨ EDDDDDDDĘH ,äčŠVŋ>ôëg]>pvírl="UĀ°yŠ†”GãPEŒAYgĻ– pwwwt ÅĶe„RÍ:"Ռō(bĘŖˆ1(‹âĖÔĀrGŽqt ÅģũöüåīžsT"UÆĐyŠf”GãPEŒAYgĻ–TŽfÍ KëōæÍpô¨cë§Ŗ– čØąŖŖK(Ų՗j–¸8ÃįQ¤QEŒCy1eQœ™áX“&MÂd2x4nܸÄ}~ûí7ĸĸĸđđđ e˖LŸ>ŊŠĒ5†„„G—P˛Ģ/#œ?ßaeˆTÃįQ¤QEŒCy1eQœ™áX$''ÛÛˇo/vÛÇ3tčPúöí˖-[øÛßūÆc=ÆüjÔ(1›ÍŽ.ĄdĐŽuyÕ*HJrl="•ČđyŠF”GãPEŒAYgfČV5hܸąíŅ°aÃbˇ>}:͚5cʔ)„……ņāƒōĀđîģīVaŎåáááčJf2ÁˆÖe‹ūķĮÖ#R‰ ŸG‘jDy1åQÄ”Eqf†l`íßŋŸ&MšĐĸE FŒÁĄC‡ŠŨvíÚĩ <¸ĀsC† aĶĻM\ž|š˛K5„ˆˆG—pm#Gæ/Īž]Ē]0™L„„„TNMÅČģtĩ´ú÷īÉd"66Ööܗ_~IûöíŠ]ģ6žžžÜwß}•PŠ‘SäQ¤šPEŒCy1eQœ™áXŨģwįĢ¯žâ—_~áŗĪ>ãĉôęՋĶ§Ošũ‰'hÔ¨Qį5jDvv6ŠŠŠE•…Ųl.đpf›7ovt ×ÖŽ]ūŨ7m‚ũû[O fĪžÍėR6ŲŠōŲgŸ1fĖ._žĖûīŋĪoŧŸŸŸ+#sŠ<ŠTĘŖˆq("Æ ,Š3ĢáčūčĻ›n˛-wčЁž={ŌĒU+žüōKžzęŠ"÷ųãh‹ÅRäķyŪzë-^}õÕBĪoÚ´‰ēuëŌĨKvīŪÍŋŠW¯-Z´`Ûļm4oŪœÜÜ\Ž;@dd$ ##ƒēuëŌļm[ļlŲ@PPîîî9r°Ūņ!!!ŗŲŒ‡‡ļ/&MšāááamÖž}{9{ö,ĩjÕ"22’ 6иqcŧŧŧ8pāŲŲŲėßŋŸ´´4jÔ¨ATT6lĀbąĐ°aCüüüØˇoíÚĩ#--S§NáææF׎]Ų´i999Ô¯_Ÿ€€vīŪ @›6m0›Íœŗ999$&&ĐšsgöíÛĮųķįņōōĸuëÖÄĮĮŒ››[ĪėáÇ9wîuęÔ!,,Œ¸+˙ ›6mJ­Zĩ8|ø0`ÍëącĮ8{ö,ĩk×ĻcĮŽlܸŅö™­[ˇ. <<œ'N––F͚5éŌĨ ëׯ ÛûJjj*ŠŠŠļĪėƍÉÍÍĨAƒ4hЀ={öØ>ŗéé餤¤úĖúûûĶ¸qcvíÚeûĖž?Ūö~wíڕmÛļ‘••…¯¯/ÁÁÁļų÷Z´hÁĨK—Hē2[Ißy˙]ŒôÆɓ' ˙‘÷~‡„„mûĖ÷ĐŦY3Ž= @§N8xđ xzzjûĖQŖF ÛäĨ:tāčŅŖ¤§§ãááAûöíŲ´ixzzÚ>ŗ?~œ3gÎúĖ^ë;"::Úö™mØ°!ūūūėŨģ€ļmÛræĖN:…ÉdŌw„ŋ#Ο?OzzēĄž#Œø÷}Gč;ĸ*ž#Ėf3—/_6Ôw„˙Ąī}GTöwę3gXŋ~Ŋáž#Œø÷}GØ˙;"00Š0Yōē=6hĐ ZˇnÍ'Ÿ|Rh]ŋ~ũčÜš3S§Nĩ=ˇpáBîžûn.\¸@͚5 퓕•EVV–íwŗŲLpp0éééx{{W΋¨DĮŽ#88ØŅe\[R4kšš‚[ņƒhŅĸ͛7gâĉ<účŖ 2„ųķįãééÉüųķIJJÂÃÃ#GŽ0yōdēwīÎęÕĢ‰eĀ€Lœ8‘ÔÔTfΜÉu×]Gll,ëׯįđáÃ\ēt‰gžy†””fĖ˜Á¸qãlĪĸĸ‘‘‘Aff&5kÖÄĮĮ‡ūũûķÛoŋąråJ""" víÚ4oŪœ}ûöQŋ~}^ũu~øaûŋ§b8N“G‘j@y1åQÄ”Eq$ŗŲŒOš{/†ģ„đ˛˛˛ØŊ{wąēž={˛lŲ˛Ī-]ē”ččč"›Wĩk×ÆÛÛģĀÙ9ÍD|M›BŪ|e đŋ˙•jˇ¤¤$yäĖâŋņôôāôéĶŧũöÛüå/áī˙;999ļŽ}žđđp&OžĖsĪ=`ëwīŪģîē‹īŋ˙ž””&MšÄ¸qãŽYËŖ>JÆ iØ°!ˇŨv[Ąõy#˛˛˛˛¸á†˜;w.îîî<ōČ#%ŪMS\‡ĶäQ¤PEŒCy1eQœ™áXO?ũ4ŋũö‡fũúõ >ŗŲĖũ÷ßĀķĪ?ĪčŅŖmÛ?ôĐC9r„§žzŠŨģw3sæLūõ¯ņôĶO;ę%Tš’&š7œ1cō—gÍ*Õ.uęÔÁ×חuëÖŲ†û^¸pņãĮÛFV-]ē”ÚĩkÛFGåÉģƒe^33;;°ŽŽzđÁ™7o&Lā•W^)U-'NdŲ˛e,[ļŒ÷Ū{¯ĐzüũũxėąĮ¸ëŽģčÛˇ/‹Å6LU\›SåQÄÅ)"ÆĄ<Šƒ˛(ÎĖp ŦÄÄDFŽIģv툉‰ĄV­ZŦ[ˇŽæ͛œœlģFŦ׋ūôĶOÄÆÆÉë¯ŋ·~ȝwŪ騗 %ší6đņ°uėž 'c!7§Ø]üũũų駟ČÎÎfđāÁÄÅÅaąX0™Ldgg“––ÆÜšs \z-/ŧđŗfÍ"44”ččhæĖ™cģ.ē$ááá 8Uä6<ō¯ŊöĶ§OgŊxyyŅŖGR×'"""""""ų 7‰ûœ9sJ\?ĢˆQ;×]w]ĄKĮĒ“öíÛ;ē„Ōķđ€ŋDΰ. "ū~Āoxy„Ū[än={ödņâÅÜ|ķÍ 4ˆ+V0mÚ4^xáŪ˙}ƍGũúõ‹Ŋ[å­Yŗ€={öØ&mŋ˙ūûéŪŊ{…_â‹/žˆŲlæÛoŋeŅĸEtęԉˇŪz‹&MšTøØb|N•G§<Š‡ō(b Ęĸ83§˜ÄŊ˛Ut"1GÛģw/íÚĩstĨŗį[¸é>8\ũÉ3́Ÿŋ)ļ‰%⠜*".Ny1åQÄ”Eq$—ŸÄ]ŽíėŲŗŽ.Ątrsāĩņ…›W\ųũÖõ%\N(btN“G‘j@y1åQÄ”Eqfj`š€Zĩj9ē„Ō9ĩ ~1n^åą`]jUUV%bWN“G‘j@y1åQÄ”Eqfj`š€ČČHG—P:“áĖ5ļ9se;'å4yŠ”GãPEŒAY”ŌX˛d ŅŅŅtėؑ=z°uëVG—¨å6lØāčJ§N uÂö’ø]ŲNÄI9MEĒåQÄ8”GcPåZΜ9Ã}÷ŨĮ×_Íļmۘ}:ņņņlÚ´‰Ö­[ķÚk¯]ķxmÚ´áÔŠSŦ[ˇ€… ’‘‘ABBB%ŋ’kĢáč¤âŧŧŧ]Bé…Ū ?cŊÛā/fëœW~@ ËZŨíØúD*ČŠō(ââ”GãPEŒAYtM#GŽdåʕ 2„+VĐ Aƒë~øaš4iÂK/ŊtÍcųøø0ū|ž{î9Ν;GŸ>}§f͚•U~Ši– 8pā€ŖK(›Đ{á›4Øĩũîčemj­Hƒ˙üĮŅՉTˆĶåQą)"ÆĄ<Šƒ˛čš˛ŗŗŲącūūū$%%X÷ú믓˜˜ČôéĶ‹Ü÷Ģ¯ž˛]føÅ_Đ¯_?bccŲŧy3˙øĮ?8~ü¸í’BGŌ,q 7whÔßē|üĢŸuųwāŪ{Ą×抈ˆˆˆˆˆ¸ĒœXĩ ’“!0úö÷"fÜyîšįh׎_~ų% **ŠÖ­[ķÕW_ņŨwßņÛoŋQŖFŅíŸŅŖG3zôčĪ%''hŊšÚë¯ŋÎõ×_OëÖ­íūúĘĘdąX,Ž.ÂŅĖf3>>>¤§§ãíííčrĘĖl6;eŨ6 ôčywĘ?bb[“H99}E\ˆō(bĘŖˆ1(‹ÎcÁxüqHLĖ.(ĻN-øŋË?üđĪ=÷6lĀĶĶ“9sæ0yōdÖŽ]k5åããcÛ>>>ūšį~đÁųũ÷ßÉÎÎĻgĪž|ôŅGøúúVø5U´÷ĸÎßĀÚŋ?mÚ´qtķãpË-ÖåđpØļ­čÖ˛ˆÁšDE\„ō(bĘŖˆ1(‹ÎaÁ>Ü:Öãjy*͛įœc>*Ú{ŅX. --ÍŅ%TÜĐĄĐŗ§uy×.˜=Ûąõˆ”“KäQÄE("ÆĄ<Šƒ˛h|99֑WE 5Ę{î‰'ŦÛU7j`š€âŽeu*&ŧųfūī¯ŧ—/;Ž‘rr‰<Š¸åQÄ8”GcPoÕĒ‚— ū‘ÅĮŽYˇĢnÔĀrQQQŽ.Á> €n°.:3g:ļ‘rp™<Š¸åQÄ8”GcP/9ŲžÛš5°\Ā†ŧÉĪ]ÁÕŖ°^23W‹H9¸TEœœō(bĘŖˆ1(‹ÆwåævÛΕ¨å\jūîŨáÖ[­ËIIđūûŽ­G¤Œ\*"NNy1åQÄ”EãëÛ×zˇÁŧ Û˙Čd‚ā`ëvՍX. aƎ.ÁžŪ|ÜÜō—KēXÄ`\."NLy1åQÄ”Eãsw‡ŠS­Ëlbåũ>eŠuģęF ,āįįįčėĢ}{xøaëō… 0qbšc2™ČČČ°caŊüōËDFF‰——-[ļ´ũžwī^ūū÷ŋĶŽ];ÜÜÜøᇠí?mÚ4ÂÂÂhßž=;v$S—Kē—ËŖˆSEŒCy1eŅ9ÄÄĀŧyĐ´iÁįƒ‚ŦĪĮÄ8Ļ.GSËėÛˇĪŅ%Øßk¯AũúÖåŲŗ y‹…×^{øøxâã㉎ŽæÃ?´ũŪŽ];n¸á~úé'úõëWhßE‹ņíˇß˛nŨ:vėØÁōåËŠYŗĻ^…Ø›KæQÄI)"ÆĄ<Šƒ˛č3fĀ–-°u+LŸ}Xĩj:ubĪž=Œ?ž˙üį?ļK÷~üņGFMā•û‘zzzâééiŸ×ōŲŲŲûŒŸ~úŠRÎ%UËeķ(℔GãPEŒAYgĻ– 8uę”ŖK¨<Ŋ{ÃX—Īƒ‡‚ ÜúÕd2Ņ˛eKvíÚō7ŪČęÕĢißž=gΜ)ÕūgĪžĩMÔ~Įw”ģŽf͚1räHÜŨŨņ÷÷įĻ›nbÆ å>ž‡KįQÄÉ("ÆĄ<Šƒ˛(ÎL ,āææâ˙ß{ŽŒŠâĮÉųvąąÖšŨccKžkæĖ™$$$đûīŋĶ§O1™LÜzë­ŧûîģX,Ž;Æ°aÃøęĢ¯8qâ.\(tĄ¯¯¯mĸö… –û%5Š%K–™™ÉoŋũF§NĘ}<1—ĪŖˆQEŒCy1eQœ™>Ŋ. k׎Ž.Ąrųú´i,bŒŽŖՀ`îåNĢÁ|ā˙:‹žÍ(r×ÚĩkĶģwoĖG}Dpp0Ûˇo§W¯^tėؑ.]ēđ˙÷tėؑ~ũúņâ‹/2xđ`:uęÄu×]WĄĄxë­ˇ bíÚĩŒ3†   Ûņž|ōINœ8Axx8QQQÜtĶMŅ%Æáōyq"ĘŖˆq("Æ ,Š33Y,¸ËE˜Íf|||HOOĮÛÛÛŅå”ŲĻM›ˆŽŽvt•nQ÷7 Ú°€Hâq'×ö|nÄIâ7ŋqÛŊ^ŦP¤úäQÄ("ÆĄ<Šƒ˛(ŽTŅŪ‹F`š€œ’ŽĄs99°WvĄæ€;šDĪūņ”x9ĄHU¨yqĘŖˆq("Æ ,Š3SËÔ¯_ßŅ%TēUĢāžŒĪ 5¯ō¸‘Ë=æOYĩęÚĮJHHĀd2bß"¯Ád2a2™JŊ}˙ūũ1™LÄÆÆÚj.ę!ÆRō(â,”GãPEŒAYgĻ– pt •.9špŧØõ&Ŧ듓ĢŽĻ˛š={6ŗgĪ.מ 6´í?{öl}ôQēuëfĪËÅd2‘‘QôdE9~ü8 (ÕļƒĻcĮŽDFFŌˇo_âããKĩßâŋyæ™gŠ\7sæL:tč@5øį?˙YhũüųķéĐĄ„‡‡“PĒsæŠyqĘŖˆq("Æ ,Š3SËėŪŊÛŅ%TēĀ@8N“b×[°ŽĪģYaYĖ˜1777úôéÃŲŗgųøãiÕĒuęÔÁĪĪAƒŲŪãØØXL&]ģveėØąøųųŅŽ];Ö¯_Ā˜1c 3f #GŽdäȑe/¨[ˇ.#FŒ°=öîŨ Pl“ÆȚ4iÂʕ+KĩíÜšsŲļmņņņL˜0x TûŨzë­ŧķÎ;EŽ‹ŠŠbîÜšŒ5ĒĐē-[ļđâ‹/ōË/ŋ°sįNÖ­[Wæ?äĢCEœ…ō(bĘŖˆ1(‹âĖÔĀ§Đˇ/üĮ{9Å|dsqã?ŪãčÛˇlĮ6m?ü0C† aéŌĨøúúŌ¸qcüqĻNĘøņãYšr%>ø`ũ6mÚD@@111ėÛˇgŸ}€‡~˜Ųŗgķå—_ÚŊzõēf¤ĻĻ’ššJzzz‰Ûnßže˖Ņ˛eKbbbĘö‚+ÉģīžKīŪŊiÛļ­m”ŲO?ũDdd¤íáááÁ—_~IBB 4°ík2™˜4iRĄũ|}}mËéééĨžíīŦYŗ>|x‘ë:uęDXXX‘ĮzīŊ÷˜0aMšX›ĨŪŪŪxzz–ęœ""""""Ryj8ēŠ¸6mÚ8ē„JįîmĻ=Iü}ßIā iĘm'>…Ü8 ŗ­?s‹ŋËF:uđõõeŨēulŨē€ .0~üxRSS™9s&K—.ĨvíÚdffØˇaÆļĻWvv6‹…|yķæ1aÂ^yå•RŊļ‰'˛lŲ2–-[fkPåøņãĖž=›úõë3vėØRģ*äPkŲ˛%}úôaÕUŗé¯\š’I“&ņã?R¯^Ŋ2ī˙ÕW_qėØ1ŪxãJŋdōōåËlŪŧ™%K–°zõjÖŽ]ˌ3ĘtŒę’Gg <Š‡ō(b Ęĸ835°\Āɓ']B•ší^/žL{‰ƒ+1÷ß9œōO2•Ûøæ> #ŧ!|´eũyŸ?ėųļČcųûûķĶO?‘ÍāÁƒ‰‹‹Ãbą`2™ČÎÎ&--šsį’••Uęú^xáf͚Ehh(ŅŅŅĖ™3Į6?VIÂÃÃ8p $**ĒØí>úč#.]ēÄøņã }i[ŪŨwėØÁØącY´h‘í˛ŧ˛ėĩûīŋŸ•+W9‚køđáļK‹Z_Z͛7įÎ;ī¤N:xzzÆ ĘtŒę”GŖSEŒCy1eQœ™XâtÜŨĄ9ú?Ū ÷įžÎ@Ēæ]€4Ŧ×ĻsĖpĶ}Å6ązöėÉâŋšxņ"ƒ b˙ūũL›6 đūûīTĻ[ÍŽYŗ€={öØ&m˙ä“O*ú’8ū<3fĖĀÃÃÃvÂʔ“ąą0{ļõgNņƒŲ˜9s& üūûīôéĶ‡¤¤$nŋũvžøâ """Jūūū…öŸ7oņņņÄĮĮWčÖĀŖFbéŌĨäæ撓“Ã˛eËčÔŠSš'""""""öa˛X,GáhfŗŌĶĶņöövt9RV—/Áđēđ}ļĩqõG&Ŧ#ŗžIˇ2L’U-X?‰‰ųĪÁÔŠđĮyãM&“'OfŅĸEœ:uŠW_}•‘#GōęĢ¯ōūûīĶĸE Ûļ¯Ŋö;v$::šÔÔÔ÷?vėwŪy'/^ÄÍ͍† ōîģīyÍúg͚ŏ?ūČ˙ûßBëžųæž{î9Μ9C­Zĩ¨[ˇ.ß˙=;w&77—§Ÿ~šŸūwwwúõëĮÔŠSm—ŠŠˆˆˆˆˆHųT´÷ĸÎßĀŠ‹‹ŖK—.Ž.ÃqNÆZ/L+a`×JhÔŋjjrb ĀđáđĮo†ŧĢúæÍ+ÜÄĒ“ÉÄšsįđōō˛Û1˙ņpčĐ!ĻOŸnˇc–VĩĪŖˆ("ÆĄ<Šƒ˛(ŽTŅŪ‹.!t—/_vt Žu1Î\c›3Wļ“åäXG^ÕÖÎ{î‰'JžœĐŅōæ!{衇rūjŸGQEŒCy1eQœ™X.ĀĪĪĪŅ%8V@¸Ö[āėHŧÆF˛jUÁË˙ČbcĮŦÛŲ‹ÅbąëčĢ7ß|“]ģv•ęRÃĘPíķ(b ĘŖˆq("Æ ,Š3SË”åîn.Ša_âmëĒ(& pëD:víĒÂâœKr)Š•vģę¨ÚįQÄ@”GãPEŒAYgĻ– Øšs§ŖKp,7wxy4§p˄õųdŦŧ˙ü3tč>XōPŖj*0ĐžÛUGÕ>"ĸ<Š‡ō(b Ęĸ835°Ä5„Ū ?cŊÛ ?Öƕ?Ößü &|cŊ@n.üë_ĐĻ <÷œšÖZÕGßžÖˇÉTĖh6“ ‚ƒ­Û‰ˆˆˆˆˆˆTŨ…įŋ ajj* 4ptƐ›§VY'l¯hŊŧĐÍŨēîÂøđCxûmHOĪßĮĪūö7xôQđđpLŨ’wB(8™{eŨ…ĐÕ("ÆĄ<Š‡ō(b Ęĸ8’îB(\¸pÁŅ%‡›;4ę!#­?ķšWžžÖW„ PĢ–õų3gā™g m[˜5ËØˇØĢ11Ö&UĶĻŸ RķĒ4”GãPEŒCy1eQœ™X. Y3j—Mũúđîģ°oÜūĐĸcĮ`ėXčÔ ž˙žāđŖj&&`åJø÷ŋ­?VķĒ4”GãPEŒCy1eQœ™XR}5onqĩu+Ü|sūķ;w­ˇBīŪ°bEĩmdšģC˙ū0r¤õ§ģûĩöŠš įŸ+''wu*î˙ƒgŸ…uë >Ũuđú뚹\JEy1åQÄ8”GcPő4–°cĮG—āúõƒ5k`ū|ĪūˇßŦë ‚ĩkWŸ8åQÄ8”GãPEŒAYgĻ– ČĖĖtt ŽÃd˛Nô´m|û­ub÷<˗C¯^0t(lÚä¸ÅДGãPEŒCy1eQœ™X.ĀĮĮĮŅ%¸ww5Ę:ÖŦYвeūēŸ†Ž]áöÛ­ķg‰\Ey1åQÄ8”GcPřŠåš5kæč\WÖ;îŲŸ}Wŋ׋Ad¤uÄÖæÍ+QŒEy1åQÄ8”GcPřŠåļoßîč\_͚đāƒ°oL›M›æ¯[¸ĸŖáĻ›`õjĮÕ(† <Š‡ō(bĘŖˆ1(‹âĖÔĀ)‹Úĩáá‡áĀ˜2ķ×-Y}ú@˙ūÖų˛tƒOģPË„„„8ē„ęĮÂO>æÍķ×ũö›õŽ…=zĀ÷ßĢ‘UÍ("ÆĄ<Š‡ō(b Ęĸ835°\@vvļŖK¨ž<<āĄ‡`˙~ëdīWßĩpøõVëŨú{žķį­#ąúôĐP˜<ŲēT åQÄ8”GãPEŒAYgĻ– ČĐ%b’§^=ëXqq°nŒg}.Īž}đÜs ˇÜ ĀĨKŽĢ×)"ÆĄ<Š‡ō(b Ęĸ835°\€§§§ŖKŖ1™Ŧsc͘ÉÉđÕWĐŋūúÜ\øņG¸ķNhÚž|ŌÚôŌÅĻ<Š‡ō(bĘŖˆ1(‹âĖ ŨĀzë­ˇ0™L<ņÄÅn‹Éd*ôØŗgOÕę`ĄĄĄŽ.AŒŦn]øŋ˙ƒ•+áāAxé%ëŦ<ŠŠÖIáŖĸ "Ū|U­ĶSEŒCy1åQÄ”Eqf†m`mܸ‘O?ũ”Ž;–jûŊ{÷’œœl{´iĶĻ’+4Ž¸¸8G— ÎĸeKxí58|~ųFŒ€Úĩķ×īŪ /ž-Z@ßžÖ\iiŽĢ× )"ÆĄ<Š‡ō(b Ęĸ83C6°222¸÷Ū{ųėŗĪđķķ+Õ>4nÜØöpww¯ä*Eœ˜ģ;  ŗg['tŸ1ÃÚ°ēÚīŋÃCAãÆpûí0odf:¤\ŠŪ ŲĀzä‘G¸ųæ›8p`Š÷éÜš3Üpà Ŧ\š˛Ämŗ˛˛0›ÍÎ,((ČŅ%ˆ3ķ÷ˇNöūŋ˙Y/üûß!,,ũåË°hÜu4jŖGÃ?@V–ÃJ62åQÄ8”GãPEŒAYgVÃŅüŅœ9sˆ‹‹cãƍĨÚ>00O?ũ”¨¨(˛˛˛øú믚ᆈĨ_ŋ~EîķÖ[oņęĢ¯z~ĶĻMÔ­[—.]ē°{÷n.^ŧHŊzõhŅĸÛļm yķæäæærėØ1"##9pāÔ­[—ļmÛ˛eËĀúåāîîΑ#GčØą# žUkë<ˇIDAT ˜Íf<<<ˆˆˆ`ķæÍ4iŌ:@ûöíILLäėŲŗÔĒU‹ČČH6lØ@ãƍņōōâĀ€uôŲūũûIKKŖFDEEąaÃ, 6ÄĪĪ}ûöĐŽ];ŌŌŌ8uęnnntíڕM›6‘““Cũúõ `÷îŨ´iĶŗŲĖɓ'čŪŊ;qqq\ž|???š4iÂΝ;hÕĒ.\ 99€ččhvėØAff&>>>4k֌íÛˇBvv6‰‰‰téŌ…={öpáÂŧŧŧhÕĒ[ˇn Yŗf=z€N:qđāA222đôô$44Ô6 6((ˆ5jpeūĻ:pôčQŌĶĶņđđ }ûölÚ´Éöšņôô´ŨF6""‚ãĮsæĖjÖŦI—.]Xŋ~=5ÂÛۛũû÷FJJ §OŸÆŨŨččh6nÜHnn. 6ÄßߟŊ{÷Đļm[Μ9ÊS§0™Ltë֍͛7“ŋŋ?5˛Ŋß­[ˇ&##ƒ'NĐ­[7âããštéžžžącĮZļlIff&Į **Š;w’™™‰ˇˇ7!!!>ŗ999ļ÷ģsįÎėÛˇķįĪãååEëÖ­‰ 88ˇąc92`žûöēy3ĖžMÍS§Ŧ!1›áë¯áë¯Éõö&ëĻ›8ŌŊ;æŽ]ißĨ ĮŽãėŲŗÔŽ]›Ž;ÚrܸqcęÖ­k{ŋÃÃÃ9qâiii…Ūī€€|||līwhh(ŠŠŠ¤ĻĻÚ>ŗyīwƒ hĐ mÎģ6mڐžžNJJJĄĪŦŋŋ?7f׎]ļĪėųķįmīw׎]ŲļmYYYøúúlûĖļhŅ‚K—.‘””dûĖ÷áããÃņãĮ õÆɓ'õĄīû|G¸šøĖ>|˜sįÎQ§NÂÂÂlīwĶĻMŠUĢ‡ļŊßUũQģvmŧŧŧ õaÄŋGč;BßUņaąX0Ôw„˙Ąī}GTöwđ#GHLL4Üw„˙ĄīûGR&‹Å8ˇ;vėŅŅŅ,]ē”N:Đŋ"##™2eJŠ3lØ0L&‹/.r}VVYW1›Í“žžŽˇˇw…^ƒ#Ŧ_ŋžîŨģ;ē qU99Ö āŋųžûŌĶ oãë wÜwß 7Ü5kVu•†Ą<Š‡ō(bĘŖˆ1(‹âHfŗŸr÷^ u áæ͛III!**Š5jPŖF ~ûí7>üđCjÔ¨ANNNŠŽĶŖG[Gĩ(ĩk×ÆÛÛģĀCDŠáîÂŦYpō$,^ ÷Ũõęåosö,|ņÜt“uÎŦ?ũÉz™ĄæĖ;0ÔŦsįÎŲ†ˇå;v,ĄĄĄ<ûėŗ´oßžTĮ>|8iiiüúë¯ĨÚžĸ]@GģpážžžŽ.CĒ›ĖLë įÎĩ6ĩ22 oSˇŽĩŠuĮ0t¨u¤–‹SEŒCy1åQÄ”Eq$—U¯^=Úˇo_āQˇn]ęׯok^=˙üķŒ=ÚļĪ”)SøîģīØŋ?;wîäųįŸgūüų<účŖŽzU.īšm‘*åáˇŨß~ ))°`ŒamZå9Ūz÷Â{ī…† aČøäë]”ō(bĘŖˆq("Æ ,Š33Ü$îגœœ\ t—.]â駟&))‰:uęÁ?ūČĐĄCXeÕJ/jN"‘ĒT§Žu”ÕwĀ… °|9,\ߧO[ˇÉΆĨK­ņãĄ{w¸ũv¸ųfhßL&‡ž{QEŒCy1åQÄ”Eqf†ē„ĐQœũ­[ˇÚ&Ŋ1”ėløũwëäī Bq˙âĶŦ™ĩ‘uķÍpũõֆ˜“REŒCy1åQÄ”Eq¤Šö^ÔĀÂųX999¸ģģ;ē ‘’Y,omd}÷\šml!uęX›Xy ­+ˇ7vĘŖˆq("ÆĄ<Šƒ˛(ŽäRs`IųlÚ´ÉŅ%ˆ\›É;Ãk¯ÁļmpāL™ƒA­ZųÛ]ŧ?ūhŊĖ°ysčØž{~ũ˛˛V~i)"ÆĄ<Š‡ō(b Ęĸ835°DÄ1Zĩ‚ĮˇÎ‰•šjūO‚ƍ nˇ};Lž 7ÜūūÖģN™ģvYGu‰ˆˆˆˆˆˆËsēIÜĨ°ĀĀ@G— R1õęåOŸ› [ļXGaũø#lܘߨēp~ūŲúhÚļ>n¸Áz§CSEŒCy1åQÄ”Eqfš įŸ+55• 8ē ‘Ę‘’bŊĢaŪ ““‹ßļK—üfV¯^āéYuu^Ą<Š‡ō(bĘŖˆ1(‹âHšK8xđ ŖKŠ<0jĖšIIÖK ß{† )|ˇÂ¸8xûmëŧZžžĐ§ŧøĸĩváB•”Ģ<Š‡ō(bĘŖˆ1(‹âĖt Ąˆ8“ Úˇˇ>žz 23aõęüŅYņņųÛ^žl]ˇz5ŧų&ÔŦ ŨģC˙ūÖGĪžĄ%""""""e§KqūK322đōōrt"Žwō¤õn…ąąÖĮž}Åo{uCĢwokCËĮ§Â%("ÆĄ<Š‡ō(b Ęĸ8RE{/j`áü Ŧ}ûöŅļm[G—!b<Įį7ŗbca˙ūâˇÍŨÕģwū#$Äú|("ÆĄ<Š‡ō(b Ęĸ8RE{/ē„Đœ9sÆŅ%ˆS“&ÖųŗF˛ūž”ŋũ+WZZäokąXį×ÚžĻOˇ>X°ĄišUåQÄ8”GãPEŒAYgĻ– ¨y˙Ą‘+š6-ØĐJL„ßĪŸ+këVČÍÍß>9æÍŗ>Ā:gVt4të–˙hÖŦĀ(-åQÄ8”GãPEŒAYgĻKqūKEÄNΝƒõëķZëÖYŸ+I@€ĩ‘ÕĩkūĪúõĢĻ^'Ą9°ėĀŲXëׯ§{÷îŽ.CÄõääX/)\ŗ&ŋŠuäČĩ÷kÕ*ŋ™eŊôĐ ŋ[Dœū|1åQÄ”Eq$́%"RYÜŨ­Í§ČH?ŪúÜɓ°qŖõąaƒõ‘–Vpŋƒ­ŲŗķŸkÕ ētÎķUÕ+qjj`š€FúŸ`‘ĒĶ¨Ür‹õÖÉ߲5ŗ2W­ÂcįNČĖ,¸_^Sëŋ˙ÍŽI“üfV—.ÖFYķæāæVe/GĕéĪGãPEŒAYgĻ– pÆËE\†Éd]ÕĒŒɅ´4<ęՃ;`Ķ&Ø˛ââ`Û6¸xąāžĮ[?ū˜˙œ—DD@ûöĐĄCūĪ€€Ē}]".@>Š‡ō(b Ęĸ835°\ĀūũûuŗˆAØō˜7˛*Ov6ėۗßĐÚ˛Åú8{ļā22Ŧɯ__đų† 6´Úˇˇ6ēęÕĢô×$âŦôįŖˆq("Æ ,Š3SKD¤*Ô¨ááÖĮŊ÷ZŸŗX !!ŋ™ĩm›uäÖĄC…÷?u ~ũÕú¸ZķæĄĄÖŅaRĸ„„Z´hAķæÍIHH¨˛ķšŽüˇ)í}Tú÷īĪoŋũÆʕ+éßŋ?ß˙=¯žú*{öėÁÍ͍ˆˆŪzë-ú÷ī_‰U‹ˆˆˆˆ8ŽX. ,,ĖŅ%ˆČeĘŖÉ-ZX11ųĪgdĀŽ]Ö; îØaũš};¤¤>Ƒ#ÖĮ’%Ÿ÷õ-ÜÔ …–-ĄfÍrŊ6ąŸŲWOđ_F/^äŽģîâōåËŧūúëœ={–wŪy‡QŖFqüøq;VéüôįŖˆq("Æ ,Š3ĶLÁ. Ĩ¨˙Š‡°KŊŧ [7øĶŸāƒ`ųrëŨOž„+`ęTxđAčŅŖøKĪž…uë`Ö,xî9¸ũvkËĶĶúsØ0xōIøøckķëĀ¸|šâĩ;š3fāææFŸ>}8{ö,ü1­ZĩĸN:øųų1hĐ vīŪ @ll,&“‰Ž]ģ2vėXüüüh׎ë¯\ū9fĖL&SĮ˜1c9r$#GŽ,W999˜L&jÕĒÅĀšūúëđ÷÷¯øābôįŖˆq("Æ ,Š3Ķ,púôiZˇníč2D„JÎc@\Ŋõ‘Įb'`ĪžÂŖG #;öîĩ>ūČŨBBŦŌˇnm}ä-ˇl •ķē bÚ´i<účŖ 2„ųķįãééIãƍyüņĮņđđāȑ#Lž<™|ÕĢWÛöÛ´i×_=111Ėœ9“gŸ}–ØØX~øanŧņF.]ēÄ3Ī (¸îüyëÄņllíÛW$ääĀÁƒÖĮŌĨEŸ'$ÄúhŪŧāĪfÍ NĘyU ))‰Gy„!C†°xņbj^šĖōôéĶŧũöÛ$''Ûļ‹‹+°oxx8“'Of˙ūũĖœ9“ĐŊ{wĸŖŖ1b)))Lš4‰qãÆ]ŗ–G}”/ŋü€ëŽģŽØØØë/_žĖk¯ŊFVVŸū9<ųä“Ü{īŊŦ_ŋ77 ŽÎŖ?EŒCy1eQœ™ÉRÚd]˜ŲlÆĮĮ‡ôôtŨVTD\_n.?nŊlđāAëĪĢ—Ī+ßq5*ØÔēz98¸ęī˜˜›§VÁÅd¨ û‚[Áŋ´åMâ^¯^=ÜŨŨąX,,_žœččh.\¸€ˇˇ7nnnĖ˜1ƒ   † FVV‹…ØØX `k2åĢQŖFœ8q‹ÅÂ<ĀŦYŗ˜0aīžûŽíŧ%Mâžk×.Û\V~~~DEE˜ÄŊnŨētë֍Ž;˛uëV5jDJJ ĮŽ#((¨˛ŪQ‘rĢhīE#°\ĀƍéÚĩĢŖËœ$nnd}üņŽu‹õŽ‡E5ļŦķp'ožŽ+s@âíŪ  hÚ´āīAAāįgŸģ'îų^ŋ˜á ā ņ†—§AčŊ…6÷÷÷göėŲ 4ˆÁƒŗ|ųrÚĩk‡Éd";;›´´4ÖŦYCVVVŠKxá…˜5kĄĄĄDGG3gÎZ´hqÍ[W‡‡‡^ėú-ZPĢV-vîÜÉäɓÉČČ %%…úõëXęúĒ§ČŖH5Ą<Šƒ˛(ÎL ,››ëčDä §ĪŖÉdk+ zö,ŧūâEëÜZ ÖĮ‘#—Kē žŲlŊģâŽ]ÅoS§NáÆVŪ%’[Ö‰î‹ŗį[¸é>8ä pJæ˜aí}đ3E6ązöėÉâŋšųæ›4h+VŦ`Ú´iŧđ ŧ˙ūûŒ7ŽúõësúôéâĪ}•5kÖXËŲŗĮ6aûũ÷ßÍÖĩ4hЀųķį3iŌ$^ũuÜŨŨéׯoŋũļ. ø§ĪŖˆ QEŒAYgĻKqūK:D˖-]†ˆ <’™ ĮŽnl%&æ?ŠšĢŦęÖÍof]ŨØj??ķ/ä7¯ŽfFxÃ7i….'×Síķ(b ĘŖˆ1(‹âHē„PtëtŠöyôđ€6mŦĸX,–II›ZüŨl.ų<įĪįO8ĩ0ā$E7¯¸ōü/f˜ōđî‘?Ú,īQˇŽ}.aC¨öy1åQÄ”Eqfj`š€Ŋ{÷Vø’ąåņL&¨_ßúčØąøíΝËoj8ÉÉ֟W/''ÃŲŗ÷ķö\Ŗ†3ĀŧÁÚ^W§NáĻVq† áĘŨ ؔGãPEŒAYgĻ–ˆˆOŊzj}”$3Ķ:q|^SëÔ˙`īÖ9¯Šã˙ßŪŊ‡EUî{˙ˇ™á~‘Ģ€@( QļĨ‰Ĩ"u<ēŨíÔÜ=zĘŨVņļĩô”ģ4kKVģÜ>,9)ĩŗėĸ–4o)ļĶT ÂIQDAšËexĪ FG.×Y3|?Īŗ†w­5ëq}ųņŽwáfëjj¤K/]Ō¯ŸNNēE-w÷ļ ^ŽŽŨEDDDDÔIœ Æ?Vii)œœœ Ũ "ķhpāOÎŌ„íḿõ¸ đäG@ņ  ¨¨årípũētšcw˛°h{$—Ģ+Đ¯ŸîcŧēŒy$’æ‘H˜E2$΁Eü"’æŅĀĖĖ—¤ģ ŪyB@*^ °ö} xZûĪŖŅ7Ú(pĩļTTÜģo Ō]ÛģSã,,¤BVsaëî×Ũm..€••~ĪŨG0DōÁ<ÉŗHƌ,P\\Ė;IÉķ(Áŗ€ŊÖ.&l/…tŲ`ŒŊTÜ žuīį07ŋ=BJ55@qą4zKŸ‚WCÃŊŸŗĄáöŧ_úrphŊĐÕVŅËÖÖ¤Gy1DōÁ<ÉŗHƌ, 0á_>ˆŒ ķ(Áŗ€gÅ?5Wĩ'ā:FĄÕÔjĀ×WZîEiōųæË‹‹ĨKīüz÷ãÚZũúQV&-wߝą-JĨî(¯Ö ]ũúŨžxßÅEÚĮH0DōÁ<ÉŗHƌs`ÁøįĀ"""&PUÕvĢĩļģīÎ؝llZĩ\\Zokn7ņ‘^DDDDtoœ‹ššŠáÇēDæ‘z€B!€lmũöŠ¯—æđēWĄëίõõú=wU•´č{§F°´ėXÁËÅEēÃŖy×FĖ1DōÁ<ÉŗHƌ,Đ Ī\*DÔ+˜G’KKĀÃCZô!P^Ū˛ĐU\,Âî^Ž_JJ¤ÉîõQ_ßņųŧ éNŒíģœĨB—ŗķíĮöö€™æ‘HN˜G"y`ɘą€eœ Ũ"jÂ<’QR(¤ ā€Ā@ũöil”Š^wļÚ*x5?ŽŠŅīų…JKĨåüyũ_‹™™TørvÆkkĀĶSˇĀÕŪc#šß‹ČØđį#‘<0‹dĖXĀ2îîî†î5aŠĪh.9:÷Ũ§˙~55+xŨ¸Ņą9ŊĨŅa%%P@F†ūûZ[ßģČÕZÛŖžˆ¨uüųH$Ė"3°L@VVĸĸĸ Ũ "ķHtOj5āí--újhFbŨYØ*)‘ÚšŠU­=7oBŅ‘{ÕTWKK~~Į^ĶŖžāätģ°wįã֖æõ*UĮŽGd„øķ‘H˜E2f,`‘ŧYXŽŽŌŌ'CTHHëEŽö `%%@m­ūēcÔW§(•í¸îUŗ˛ęÜq‰ˆˆˆŒ X& PßųJˆ¨Į1Dō$yœœ:v™# ]ę¨Į(¯Ëʤųģ:ĸļ¸vMZ:C­nģĀÕ<ˇ™ƒƒtŠãŨííĨĨ‹w|$ēū|$’f‘Œ X& ˛˛...†îy$’“.åQ­ú÷—–Žhl**¤ĸÖ͛m/m­//īx_kj¤åęՎīÛĖÖVˇ¨Õ^ÁĢ­6ĩZē!Q+øķ‘zKnn.üũũ1`ĀäææöÚqMŸBĪ?bDGG#99‡Ftt4˛˛˛°lŲ2?~ …SĻLÁƍaooß­ũdɘą€e 1`ĀCwƒˆĀ<ɉAōhfvģĀĶTÄj¯ČÕ^!Ŧ˛˛sĮ­Ŧ”–‚‚ÎíH—zę[ŗŗ“Šfvv­/frøķ‘LŨ§Ÿ~Úé}kkk‹+WŽāõ×_GNNŪ{ī=!đá‡vc/™E2n,`ɅšųíËũũ;žCƒt㝎ōrŠ­ų띏[[×Ų"XCC×æģ“ZŨvqĢŖ‹ G†õaīŋ˙>æĪŸx˙÷˙‡mÛļáíˇßFAAT*"##ąqãF„„„āȑ#7n"##†¯žú nnnøčŖ…9sæ´((͞=III˜9s&`ƌîcff&.]ē„!C†āšįžÃ­[ˇđŪ{īáã?ÆƍáĐŲ?Š™°LĀȑ# Ũ"jÂ<ÉGŸĖŖ…āâ"-ĨŅH—Aę[đj­­Ŧ ¨¯ī|š/‹,*ęüs4S(ÚíÕ\䲹‘ļĶ÷ą˙Ũ}2dp X¸p!bbb°cĮX[[ÃÃÃK–,JĨÂĨK—°~ũzĖ;?ūøŖvŋ””<üđØ6mļlŲ‚•+Wâȑ#˜?>&Mš„ēē:<˙üķ(** ܸqC¯.;vLęĘŲŗÚIÛgĪž}Ī–>Ž_ŋŽÄÄD”——cĀ€xë­ˇ°lŲ˛.?īŨøŗ‘Œ™Bt÷EĩF¨ŧŧ(++ƒŊŊŊĄģĶa'Nœč–M"ę:æ‘H>˜G2)Ōŧbwļ:û¸ŊuĄ_­áXZęļŦŦ¤6KKiâū;ŋļõXßļŽėcn.Z37o{ioŊ>ûĘšxwvû'ā€;{U`īĮmąH6’!uĩöÂXDDDDDrgfv{äXWî2ŠúzŠU[+Íēa))(€ŗĩĩT(ģ×ö†T_ßĩ›3…ĸũâWs‘Ģ3K—öā–Ö˛x…Ļī/A™õņŒ6/'$"ĶĀ– 0tˆ¨ ķH$Ė#Q'YZŨú”šâb ibėv !Mß\ĖĒĢģŊÔÖöūãÚZ ĄA*j55Õ—Ōklš°\6BœCËâU3é˛Ââ÷č^떱âĪF2f,`™€ģīšAD†Ã<ÉķH$zįQĄ¸=÷U7Ņē•FŖ[Ôēŗ¸Õ^[g÷Ņh¤ËH5šļ—öÖwvŨëĨ"Wg—Îîī(€ŗ÷ø÷(…4ą;Ũ6’1cËĀĮĮĮĐŨ "0DrÂ<ɇÉåąųō:ęy׎Ąã€’vļq‚tWBē'“Ë"õ)ŧ-ɓëénƒmÍ1¯€´ŪuLoöŠˆ €w!„ņ߅°ĄĄLG$Ė#‘|0DōÁ"ų`‰äƒy$’æ‘ē,xđņ énƒ5WĨ9¯\Įfœ‹Ŧ#˜E2f,`™cŧė‘ČT1DōÁ<ÉķHŨÂĖp6t/ŒŗHƌ—š???Cwˆš0DōÁ<ÉķH$Ė"3°L@FF†Ąģ@DM˜G"ų`‰äƒy$’f‘Œ XDDDDDDDD$k,`™€ē DԄy$’æ‘H>˜G"y`ɘą€e4Ąģ@DM˜G"ų`‰äƒy$’f‘Œ X&āʕ+†î5a‰äƒy$’æ‘H˜E2f,`‘Ŧ)„ÂН0´ōōr888 ŦŦ ööö†îN‡ÕÕÕÁĘĘĘĐŨ "0DrÂ<ÉķH$Ė"RWk/e~ûí7Cwˆš0DōÁ<ÉķH$Ė"3°L@UU•Ąģ@DM˜G"ų`‰äƒy$’f‘Œ X&ĀÖÖÖĐ] ĸ&Ė#‘|0DōÁ<ÉŗHƌ,hč.Qæ‘H>˜G"ų`‰äY$cÆ– HOO7tˆ¨ ķH$Ė#‘|0DōĀ,’1“u+>> …K—.mwģääd >*• xīŊ÷z§ƒDDDDDDDDÔãd[Ā:uę6oŪŒđđđvˇģxņ"}ôQŒ3?˙ü3^|ņE,^ŧ;vėčĨžžĄģ@DM˜G"ų`‰äƒy$’f‘Œ™, X•••˜5káääÔîļīŊ÷|}}ąaÄ„„`îÜšxúé§ņÖ[oõRo ĪĖL–˙ŒD}ķH$Ė#‘|0DōĀ,’1“åŲ‡Į{ ãĮŋįļĮĮĉuÚbbb’’‚úúúžęĸŦ\ētÉĐ] ĸ&Ė#‘|0DōÁ<ÉŗHÆĖÂиÛöíۑ––†S§Néĩ}aa!ÜŨŨuÚÜŨŨŅĐЀëׯÃĶĶŗÅ>ĩĩĩ¨­­Õ~_^^ŪĩNQ‘UëōåËX˛d öīß•JĨ÷~ …Bį{!DĢíÍâããņĘ+¯´hOII """••…ššØŲŲÁß߀ ąą—/_ :įĪŸGee%lll0hĐ üüķĪooo˜››kĢÜáááČÍÍEyy9T*ŒÔÔT€——T*.\¸ Õ+WpķæMXYYačĐĄ8yō$ĀÃÃļļļ8ū< įΝCII ,,,0|øpœ đķķCCCŽ\šˆˆˆĀŲŗgQ]] [[[Üwß}øå—_žžž€ŧŧ<Ā!C““ƒĘĘJX[[#88iiiÚ÷Ûššš€ûīŋyyy(++ƒJĨBXXRRRžžž°ļļFNN`đāÁ(((@ii),--'N ĸööö8wî $$EEE¸qãĖÍ͉S§NĄąąŽŽŽpvvFvv6`Đ A(--Eqq1 FŽ‰ÔÔT444ĀŲŲîîîÚ÷;00•••(,,Œ9ééé¨ĢĢƒŖŖ#ŧŊŊņë¯ŋj˙ÍoŨē…‚‚ĀđáÃqæĖÜēu öööđķķĶ9g5öũ6l~ûí7TUUÁÖցÚģ‘øøøĀĖĖLįœŊxņ"*** VĢĸ}ŋû÷ī+++\ŧxQû~_ž|7oŪ„RŠDxx¸ļíááíûŠÂÂB”””´xŋŨÜÜāāā }ŋƒƒƒqũúu\ŋ~]{Î6ŋßũúõCŋ~ũpöėYí9[VV†ĸĸĸįŦŗŗ3<<<™™Š=gĢĒĒ´ī÷ˆ#‘‘ÚÚZ8::ÂĮĮG{ÎúûûŖŽŽųųųÚsļ­ĪČę3"$$׎]ãg?#úÜg„ƒƒĘĘĘdõ!Į˙Gđ3‚ŸŊņaii‰úúzY}FČņ˙üŒāgDOFh4œ8qBvŸrü?#ē˙3ĸĩFĄÍÕøęĢ¯đûß˙æææÚ6F…B333ÔÖÖęŦ€‡zÆ Ã?˙ųOmÛŽ]ģđÄO ēē–––-ŽĶÚ,”••ÁŪŪž^YĪĘĖĖDhh¨ĄģAD`‰ä„y$’æ‘H˜E2¤ōōrí;S{‘ÕŦGyD[…lö_˙õ_Æʕ+[¯`Ô¨Qøæ›otÚöīߏČČČV‹W T*ĄT*ģ¯ãVQQač.Qæ‘H>˜G"ų`‰äY$c&Ģ–ÂÂÂtÚlllāââĸmᅐŸŸ>ú0oŪ<ŧûîģXļlūüį?ãøņãøāƒđ駟öz˙ E­Vē DԄy$’æ‘H>˜G"y`ɘÉō.„íšzõĒöe@ē^tĪž=8r䆊W_}7nÄūđö˛w…„„ē DԄy$’æ‘H>˜G"y`ɘÉj,Céęu˜†vâÄ DEEēDæ‘HN˜G"ų`‰äY$C2Š9° Ĩš†W^^nāžtNUU•ŅöČÔ0DōÁ<ÉķH$Ė"Rķš×ŲqT,`áöDv>>>î ‘éĒ¨¨€ƒƒC‡÷ã%„QPP;;;( Cw§CĘËËáããƒË—/ååDĻ„y$’æ‘H>˜G"y`ÉЄ¨¨¨€——ĖĖ:>%;G`033ƒˇˇˇĄģŅ%öööü"’ æ‘H>˜G"ų`‰äY$CęĖČĢfFwB""""""""ę[XĀ"""""""""YcËČ)•JŦ^ŊJĨŌĐ]!ęķ˜G"ų`‰äƒy$’f‘Œ'q'""""""""Yã,""""""""’5°ˆˆˆˆˆˆˆˆHÖXĀ"""""""""YcËČ%$$Āßß*• ÇĮ?ü`č.™ŧŖGbōäÉđōō‚BĄĀW_}Ĩŗ^5kÖĀËË jĩŅŅŅ8sæŒa:KdÂâãã1bÄØŲŲÁÍÍ S§NEvvļÎ6Ė#QīØ´iÂÃÃaoo{{{Œ5 {÷îÕŽg‰ #>> …K—.Õļ1dŦXĀ2bŸ}ö–.]ŠUĢVáįŸƘ1c‹ŧŧ>Ū€=#ę; víڅŠS§ūĸååå…ĨK—båʕ€ÚÚZ¸ģģcũúõøË_ūbĀŪ™ļââb¸šš!99=ôķHd`ÎÎÎxķÍ7ņôĶO3‹DŊŦ˛˛HHHĀk¯Ŋ†ĄC‡bÆ üŲHF#°ŒT]]RSS1qâDö‰'âØącę]ŧx………:ŲT*•;v,ŗIÔÃĘĘĘHŋ4Ė#‘Ąh4lßžUUU5jŗHdqqqxėąĮ0~üxv摌™…Ą;@sũúuh4¸ģģë´ģģģŖ°°Đ@Ŋ"ĸæüĩ–ÍK—.ĸKD}‚˖-ÃčŅŖ€y$ęm§OŸÆ¨QŖpëÖ-ØÚÚb׎] ÕūRĖ,õŽíÛˇ#-- §NjąŽ?ɘą€eä …Î÷BˆmDÔû˜MĸŪĩpáBdddāß˙ūw‹uĖ#Qī Bzz:nŪŧ‰;v`öėŲHNNÖŽg‰zŪå˗ądÉėßŋ*•ĒÍí˜G2Fŧ„ĐHõë×æææ-F[ĩ¨ĻQīņđđf“¨-Z´ģwīÆáÇáíí­mg‰z—••‰øøx 2˙üį?™Eĸ^”ššŠĸĸ" >°°°@rr26nÜ mæ˜G2F,`)+++ >Đi?pāxāõŠˆüũũááᡓÍēē:$''3›DŨL… bįΝøūûīáīī¯ŗžy$2,!jkk™Eĸ^ôČ#āôéĶHOO×.‘‘‘˜5kŌĶĶĀ<’Ņâ%„FlŲ˛exꊧ‰QŖFaķæÍČËËÃŧyķ Ũ5"“VYY‰ķįĪkŋŋxņ"ŌĶĶáėė ___,]ēëÖ­ÃĀ1pā@Ŧ[ˇÖÖÖxōÉ' Øk"Ķ‡O>ų_ũ5ėėė´Mvpp€Z­†BĄ`‰zɋ/žˆØØXøøø ĸĸÛˇoĮ‘#GđŨwß1‹DŊČÎÎN;d3¸¸¸hۙG2V,`ąéĶ§ãƍXģv-Ž^ŊŠ°°0ėŲŗ 0t׈LZJJ ƍ§ũ~Ų˛e€Ųŗg#)) +VŦ@MM ,X€ŌŌRDEEa˙ūũ°ŗŗ3T—‰LŌĻM›ŅŅŅ:í[ˇnŜ9s€y$ę%׎]ÃSO=…ĢW¯ÂÁÁáááøîģī0aÂĖ"‘œ0dŦBačNĩ…s`‘Ŧą€EDDDDDDDD˛ÆÉ XDDDDDDDD$k,`‘Ŧą€EDDDDDDDD˛ÆÉ XDDDDDDDD$k,`üüü°aÃö!77 …éééíõ=<˙ėėlxxx ĸĸĸĮUTTWWWäįį÷øąˆˆČt°€EDD}Raa!-Z„€€(•Jøøø`ōäÉ8t营ģf§NÂŗĪ>ÛcĪ?gÎ(Šv\Ŋzaaa=֏ž#FĀÎÎnnn˜:u*˛ŗŗģũ8ŅŅŅÚ÷KŠTĸ˙ū˜ø ņĘ+¯ <<õõõØˇo6oŪŒŗgĪē‹-Ô××ÃŌŌŌĐŨč´˛˛2ÔÔÔhŋ÷ôôÄÖ­[1iŌ$m›‡‡‡!ēÖ*???$%%!::Z¯í'Mš„3f`ĈhhhĀĒUĢpúôidffÂÆÆĻÛúAƒaíÚĩ¨¯¯G~~>víڅwŪysæĖÁæ͛ģíXrÂķŋg]šr¸páŧŊŊ{嘧OŸÆȑ#QPP''§^9&7ŽĀ""ĸ>gÁ‚P(8yō$üq 4ƒÆ˛eËđĶO?iˇËËËÔ)S`kk {{{<ņĸvíšvũš5k0tčPlŲ˛žžž°ĩĩÅüųķĄŅhđÆoĀÃÃnnnøûß˙Žs|…BM›6!66jĩūūūøâ‹/´ë›/%úüķĪ •J…?ū°uëV„„„@ĨR!88 Úũęęę°páBxzzBĨRÁĪĪņņņ:ũõõõ…RŠ„——/^Ŧ]w÷%Túžöũë_đķķƒƒƒfĖ˜ŅæåGđđđĐ.āčč¨Ķv÷%TGŽBĄĀž}û0lØ0¨Õj<üđÃ(**ÂŪŊ{{{{Ėœ9ÕÕÕÚc !đÆo jĩC† Á—_~Ųö Ņ žûî;Ė™3ƒƐ!C°uëVäåå!55ĩÕí…?~<&Mš„æŋ%ŪŧyžžžXĩjUģĮ˛ļļ†‡‡|||đģßũëׯĮûīŋÄÄDĻOŸ'''¸¸¸`ʔ)ČÍÍÕyŽ-[ļ`đāÁP*•đôôÄ… ĩëxū÷ķ˙ķĪ?Į!CtŠW—.]Âäɓáää <{öėŅŽĪĖĖÄŖ> [[[¸ģģ㊧žÂõë×ĩëą~ũzBŠTÂ××Wį\¸˙ūûááá]ģvĩÛ7"""-ADDԇܸqC( ąnŨēvˇkllÆ ŖG)))⧟~bėØąÚmV¯^-lmmÅã?.Μ9#vīŪ-ŦŦŦDLLŒX´h‘8{öŦØ˛e‹ Ž?ŽŨ€pqq‰‰‰";;[üíoæææ"33S!Äŋáįį'vėØ!.\¸ ōķķÅæ͛…§§§ļmĮŽÂŲŲY$%% !„xķÍ7…8zô¨ČÍÍ?üđƒøä“O„B|ņÅÂŪŪ^ėŲŗG\ētIœ8qBlŪŧYÛ§ˆwŪy§Ã¯}Ú´iâôéĶâčŅŖÂÃÃCŧøâ‹zũ;ģvíŌik~Ũ?˙üŗBˆÃ‡ âwŋûø÷ŋ˙-ŌŌŌD`` ;vŦ˜8qĸHKKG...âõ×_×>Ī‹/ž(‚ƒƒÅwß}'rrrÄÖ­[…RŠGŽŅĢoÍīĮáÇõŪūnįΝÄéĶ§ÛÜæʕ+ÂÉÉIlØ°A!ÄôéĶEdd¤¨ĢĢksŸącĮŠ%K–´h×h4ÂÉÉIĖŸ?_!DUU•8p xúé§EFF†ČĖĖO>ų¤ ĩĩĩB!„JĨ6lŲŲŲâäɓ:xū˙ų?eĘ1oŪ<ļĮ{LL˜0AdddˆœœņÍ7߈ääd!„ĸ_ŋ~â…^YYY"--ML˜0AŒ7Nģ˙Š+„“““HJJįĪŸ?üđƒHLLÔ9ÆOŌžO­™7ožÎöyyyˆmŅĻ… "##Ÿ~úé=ˇũã˙ˆiĶĻ!>>˙øĮ?0hĐ ŊŽŅ!„öŧNMMÅųķįagg§íŋŗŗ3nŨē…œœĄ  <ōHĢĪÅķŋo˙555PŠT:m‹/Æk¯Ŋ†|ĢW¯FFF†v]jj*>ŦsŒāā`@NN˛˛˛P[[ÛæųÕL­Vë\ūHDDÔ Cw€ˆˆ¨7 8 …YYY˜:uj›ÛŨY h¯ũî‰Ĩ EĢm÷ėÛŨĮģsōīæũĨŗšš9 ""/^ÄŪŊ{qđāA<ņÄ?~<žüōKøøø ;;ĀÁƒą`Áŧųæ›HNNnŅߎŧv}^gGŨyœ{ŊŋÍ_ŋũö[ôīß_g;ĨRŲæ1ÖŽ]‹įž{Nû}tt4Ö¯_¯ķ^{yyŨŗ¯‹-ÂîŨģqôčQŊ&ÃŽŽŽFjj*ĖÍÍqîÜš{nߍFƒsįÎaĈ¤÷aøđáØļm[‹m]]]u L­áųߡÎ˙~ũúĄ´´T§mîÜšˆ‰‰Áˇß~‹ũû÷k‹Ŧ‹-Bcc#&OžŒõëסx.OOO\¸pA¯×VRRWWWŊļ%""â,""ęSœƒ˙ųŸ˙AUUU‹õ7oŪ ¸ČËËÃå˗ĩë233QVV†.÷ãÎÉâ›ŋoÁĐwwwôīß.\@`` Îâīī¯ŨÎŪŪĶ§OGbb">ûė3ėØą%%%¤Ņ˙ųŸ˙‰7âȑ#8~ü8NŸ>ŨâX=ũÚ{Rhh(”J%ōōōZŧOwŽ¨š››››Îļčßŋ‹ļļ!°páBėÜšß˙ŊÎŋI{–/_333ėŨģ7nÄ÷ßßá× ~ø!JKKņ‡?ü€TĖ9wî\‹×ØŲŲÁĪĪ‡jõųxū÷­ķØ°aČĖĖlŅîããƒyķæaįΝXž|9Hįי3gāįį×â86668p Ôju›įWŗ_ũÆ ëڋ&"ĸ>ƒ#°ˆˆ¨ĪIHHĀ<€‘#GbíÚĩGCC8€M›6!++ ãĮGxx8f͚… 6 ĄĄ ,ĀØącu.męŦ/žø‘‘‘=z4ļmۆ“'Oâƒ>hwŸ5kÖ`ņâÅ°ˇˇGll,jkk‘’’‚ŌŌR,[ļ īŧķ<==1tčP˜™™á‹/ž€‡‡‘””Fƒ¨¨(X[[ã_˙úÔj5 Đâ8=ũÚ{’ž{î9üõ¯Ecc#Fōōr;v ļļļ˜={v7..Ÿ|ō žūúkØŲŲĄ°°€t÷9ĩZŨę>ß~û-ļlŲ‚ãĮ#""˙ũß˙Ųŗg###NNNmĢēē………hhh@~~>vî܉wŪyķįĪĮ¸qãŗfÍ›ož‰)SĻ`íÚĩđööF^^vî܉įŸŪŪŪXŗf æ͛777ÄÆÆĸĸĸ?ūø#-ZÄķŋ˙111˜;w.4vDÛŌĨK‹AƒĄ´´ß˙Ŋļ€‡ÄÄDĖœ9Ī?˙<úõë‡ķįĪcûöíHLL„JĨÂʕ+ąbÅ XYYáÁDqq1Μ9ƒgžyĀíŅ‡ëÖ­ë7‡ˆˆŒG`QŸãīī´´4Œ7˗/GXX&L˜€C‡aĶĻM¤Ëržúę+899ᡇÂøņã€Ī>ûŦ[úđĘ+¯`ûöíĮ‡~ˆmÛļ!44´Ũ}æΝ‹˙ũß˙ERRîŋ˙~Œ;IIIÚ(ļļļXŋ~="##1bÄäææbĪž=033ƒŖŖ#ņāƒ"<<‡Â7ß|—Įéé×ŪĶ^}õUŧüōˈGHHbbbđÍ7ßč=*Ē36mڄ˛˛2DGGÃĶĶSģ´õžã™gžÁš5kXŊz5ŧŧŧ0oŪŧv•˜˜OOOÜwß}øũīĖĖL|öŲgHHHĐncmmŖGÂ××ĶĻMCHHž~úiÔÔÔĀŪŪ0{öllØ° Ų+Į#""ãĮK‰ˆˆˆˆú¸gŸ}ĨĨĨ¨¨¨ĐšģbO(**Âã?Ž™3göčqˆˆČ´°€EDDDDÔĮYXX`ÕĒUŊr,777ŦXąĸWŽEDDĻƒ—‘Ŧqw""""""""’5°ˆˆˆˆˆˆˆˆHÖXĀ"""""""""Yc‹ˆˆˆˆˆˆˆˆd,""""""""’5°ˆˆˆˆˆˆˆˆHÖXĀ"""""""""Yc‹ˆˆˆˆˆˆˆˆd,""""""""’5°ˆˆˆˆˆˆˆˆHÖXĀ"""""""""Yc‹ˆˆˆˆˆˆˆˆd,""""""""’5°ˆˆˆˆˆˆˆˆHÖūąJ2ūæIENDŽB`‚kanzi-cpp-2.6.0/doc/kanzi.1000066400000000000000000000245221525576310500153740ustar00rootroot00000000000000.TH "KANZI" "1" "Oct 2026" "kanzi 2.6" "User Commands" .SH "NAME" \fBkanzi\fR \- Compress, decompress, and inspect \.knz files .SH "SYNOPSIS" \fBkanzi\fR [\fIOPTIONS\fR] [\-c|\-d|\-y] [\-i \fIINPUT\-FILE\fR] [\-o \fIOUTPUT\-FILE\fR] .SH "DESCRIPTION" \fBKanzi\fR is a modern, modular, portable and efficient lossless data compressor\. Modern algorithms are implemented and multi-core CPUs can take advantage of the built-in multi-threading. An entropy codec and a combination of transforms can be provided at runtime to best match the kind of data to compress. The code is optimized for efficiency (trade-off between compression ratio and speed)\. Unlike the most common lossless data compressors, \fBKanzi\fR uses a variety of different compression algorithms and supports a wider range of compression ratios as a result\. \fBKanzi\fR is multithreaded by design and uses several threads by default to compress or decompress blocks concurrently\. It is not compatible with standard compression formats such as zip, gz, zstd, br, lz4, xz\. \fBKanzi\fR is a lossless data compressor, not an archiver\. It uses checksums (optional but recommended) to validate data integrity but does not have a mechanism for data recovery. It also lacks data deduplication across files\. \fBKanzi\fR generates a bitstream that is seekable (one or several consecutive blocks can be decompressed without the need for the whole bitstream to be decompressed)\. .SH "OPTIONS" .SS "Operation Mode" Help\. \fB-h, --help\fR display this message Compression mode\. \fB-c, --compress\fR Compress mode \fB-i, --input=\fR Optional name of the input file or directory or 'stdin'. If omitted, input is read from stdin. When the source is a directory, all files in it will be processed. Provide /. at the end of the directory name to avoid recursion (e.g., myDir/. => no recursion) \fB-o, --output=\fR Optional name of the output file or directory (defaults to if input is or 'stdout' if input is 'stdin') or 'none' or 'stdout'. 'stdout' is not valid when the number of jobs is greater than 1 \fB-b, --block=\fR Size of blocks (default 4|8|16|32 MiB based on level, max 1 GiB, min 1 KiB) 'auto' means that the compressor derives the best value based on input size (when available) and number of jobs \fB-l, --level=\fR Set the compression level [0..9] Providing this option forces the entropy codec and transform. Defaults to level 3 if not provided. See the definitions of the transforms and entropy codecs in the last section. 0 = NONE&NONE (store) 1 = LZX&NONE 2 = DNA+LZ&HUFFMAN 3 = TEXT+UTF+PACK+MM+LZX&HUFFMAN 4 = TEXT+UTF+EXE+PACK+MM+ROLZ&NONE 5 = TEXT+UTF+BWT+RANK+ZRLT&ANS0 6 = TEXT+UTF+BWT+SRT+ZRLT&FPAQ 7 = LZP+TEXT+UTF+BWT+LZP&CM 8 = EXE+RLT+TEXT+UTF+DNA&TPAQ 9 = EXE+RLT+TEXT+UTF+DNA&TPAQX \fB-e, --entropy=\fR entropy codec [None|Huffman|ANS0|ANS1|Range|FPAQ|TPAQ|TPAQX|CM] \fB-t, --transform=\fR transform [None|BWT|BWTS|LZ|LZX|LZP|ROLZ|ROLZX|RLT|ZRLT] [MTFT|RANK|SRT|TEXT|MM|EXE|UTF|PACK] e.g., BWT+RANK or BWTS+MTFT (default is BWT+RANK+ZRLT) \fB-x, -x32, -x64, --checksum=\fR Enable block checksum (32 or 64 bits). During decompression data is verified against the checksum in each block. -x is equivalent to -x32. \fB-s, --skip\fR copy blocks with high entropy instead of compressing them \fB--rm\fR Remove the input file after successful (de)compression. If the input is a directory, all processed files under the directory are removed. Decompression mode\. \fB-d, --decompress\fR Decompress mode \fB-i, --input=\fR Optional name of the input file or directory or 'stdin'. If omitted, input is read from stdin. When the source is a directory, all files in it will be processed. Provide /. at the end of the directory name to avoid recursion (e.g., myDir/. => no recursion) \fB-o, --output=\fR Optional name of the output file or directory (defaults to if input is , or otherwise, or 'stdout' if input is 'stdin') or 'none' or 'stdout'. 'stdout' is not valid when the number of jobs is greater than 1. \fB--from=blockId\fR Decompress starting from the provided block (included). The first block ID is 1. \fB--to=blockId\fR Decompress ending at the provided block (excluded). \fB--rm\fR Remove the input file after successful (de)compression. If the input is a directory, all processed files under the directory are removed. Info mode\. \fB-y, --info\fR Display information about compressed files \fB-i, --input=\fR Mandatory name of the compressed input file. When the source is a directory, all files in it will be processed. Provide /. at the end of the directory name to avoid recursion (e.g., myDir/. => no recursion) Operation modifiers\. \fB-j, --jobs=\fR Maximum number of jobs the program may start concurrently (default is half the available cores, maximum is 64) \fB-v, --verbose=\fR 0=silent, 1=default, 2=display details, 3=display configuration, 4=display block size and timings, 5=display extra information Verbosity is reduced to 1 when files are processed concurrently Verbosity is reduced to 0 when the output is 'stdout' \fB-f, --force\fR Overwrite the output file if it already exists \fB--skip-links\fR Skip symbolic links \fB--skip-dot-files\fR Skip dotfiles .SS "Examples" Compress recursively all files under 'dir' in test mode (no output file) using a 4 MiB block, compression level 4, and extra verbosity. kanzi -c -i dir -o none -b 4m -l 4 -v 3 Compress foo.txt to foo.txt.knz (overwrite it if it already exists) using the BWT, MTFT, and ZRLT transforms, the FPAQ entropy codec, and 4 threads; generate a checksum for each 4 MiB block. kanzi -c -i foo.txt -f -t BWT+MTFT+ZRLT -b 4m -e FPAQ -j 4 -x Compress from stdin (--input option is omitted) to foo.knz using compression level 2, 64 KiB blocks, and the default number of threads. cat foo.txt | kanzi -c -o foo.knz -l 2 -b 64k Decompress foo.txt.knz to foo.txt using 2 threads. kanzi -d -i foo.txt.knz -j 2 Decompress foo.txt.knz to stdout and delete the compressed file. kanzi -d -i foo.txt.knz -o stdout --rm Decompress foo.txt.knz to foo.txt (overwrite it if it already exists) from block 5 to block 11, using 8 threads and extra verbosity. kanzi -d -i foo.txt.knz -o foo.txt -f -j 8 --from=5 --to=11 -v 4 .SS "Transforms" BWT: Burrows-Wheeler Transform is a transform that reorders symbols in a reversible way that is more amenable to entropy coding. This implementation uses a linear time forward transform and parallel inverse transform. BWTS: Burrows-Wheeler Transform by Scott is a bijective variant of the BWT. LZ: Lempel-Ziv implementation of the dictionary-based LZ77 transform that removes redundancy in the data. LZX: Lempel-Ziv Extra. Same as above with a bigger hash table and more match searches. LZP: Lempel-Ziv Prediction can be described as an LZ implementation with only one possible match (no offset is emitted). RLT: Run-Length Transform is a simple transform that replaces runs of similar symbols with a compact representation. ZRLT: Zero Run-Length Transform. Similar to RLT but only processes runs of 0. Usually used post-BWT. MTFT: Move-To-Front Transform is a transform that reduces entropy by assigning shorter symbols to recent data (like an LRU cache). Usually used post-BWT. RANK: Rank Transform is a transform that reduces entropy by assigning shorter symbols based on symbol frequency ranks. Usually used post-BWT. EXE: A transform that reduces the entropy of executable files (X86 & ARM64) by replacing relative jump addresses with absolute ones. TEXT: A text transform that uses a dictionary to replace common words with their dictionary index. ROLZ: Reduced Offset Lempel-Ziv is an implementation of LZ that replaces match offsets with indexes, creating a more compact output with slower decoding speeds. ROLZX: Extended ROLZ with more match searches and a more compact encoding. SRT: Sorted Rank Transform is a transform that reduces entropy by assigning shorter symbols based on symbol frequency ranks. Usually used post-BWT. MM: Multimedia transform is a fast transform that removes redundancy in correlated channels in some multimedia files (e.g., wav, pnm). UTF: A fast transform replacing UTF-8 codewords with aliases based on frequencies. PACK: A fast transform replacing unused symbols with aliases based on frequencies. DNA: Same as PACK but triggered only when DNA data is detected. .SS "Entropy codecs" Huffman: A fast implementation of canonical Huffman. Both encoder and decoder use code tables and multiple streams to improve performance. RANGE: A fast implementation of a static range codec. ANS: Based on Range Asymmetric Numeral Systems by Jarek Duda (specifically an implementation by Fabian Giesen). Works in a similar fashion to the Range codec but uses only one state instead of two, and encodes in reverse byte order. FPAQ: A binary arithmetic codec based on FPAQ1 by Matt Mahoney. Uses a simple adaptive order 0 predictor based on frequencies. CM: A binary arithmetic codec derived from BCM by Ilya Muravyov. Uses context mixing of counters to generate a prediction of the next bit value. TPAQ: A binary arithmetic codec based initially on Tangelo 2.4 (itself derived from FPAQ8). Uses context mixing of predictions produced by one-layer neural networks. The initial code has been heavily tuned to improve compression ratio and speed. Slow but usually excellent compression ratio. TPAQX: Extended TPAQ with more predictions and more memory usage. Slowest but usually the best compression ratio. .SH BUGS Report bugs at: https://github.com/flanglet/kanzi-cpp/issues .SH AUTHOR Frederic Langlet .SH REPORTING BUGS https://github.com/flanglet/kanzi-cpp kanzi-cpp-2.6.0/lib/000077500000000000000000000000001525576310500141725ustar00rootroot00000000000000kanzi-cpp-2.6.0/lib/.gitignore000066400000000000000000000001641525576310500161630ustar00rootroot00000000000000**/Debug/** **/Release/** **/*.obj **/*.o **/*.htm **/*.exe **/*.idb **/*.pdb **/*.ncb **/*.sln **/*.suo **/*vcproj*kanzi-cpp-2.6.0/msvc/000077500000000000000000000000001525576310500143745ustar00rootroot00000000000000kanzi-cpp-2.6.0/msvc/Kanzi_VS2008.sln000066400000000000000000000015721525576310500171150ustar00rootroot00000000000000īģŋ Microsoft Visual Studio Solution File, Format Version 10.00 # Visual C++ Express 2008 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Kanzi_VS2008", "Kanzi_VS2008.vcproj", "{A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 Release|Win32 = Release|Win32 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Debug|Win32.ActiveCfg = Debug|Win32 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Debug|Win32.Build.0 = Debug|Win32 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Release|Win32.ActiveCfg = Debug|Win32 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Release|Win32.Build.0 = Debug|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection EndGlobal kanzi-cpp-2.6.0/msvc/Kanzi_VS2008.vcproj000066400000000000000000000342571525576310500176320ustar00rootroot00000000000000 kanzi-cpp-2.6.0/msvc/Kanzi_VS2022.sln000066400000000000000000000050361525576310500171100ustar00rootroot00000000000000Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.5.33516.290 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{B04B8A8D-9F64-4F00-A01D-BE3392588862}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Kanzi", "Kanzi_VS2022.vcxproj", "{A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Libkanzi", "Libkanzi_VS2022.vcxproj", "{3E5CE550-982F-42A4-B631-0C65582F1EAF}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 Debug|x64 = Debug|x64 Release|Win32 = Release|Win32 Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Debug|Win32.ActiveCfg = Debug|Win32 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Debug|Win32.Build.0 = Debug|Win32 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Debug|x64.ActiveCfg = Release|x64 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Debug|x64.Build.0 = Release|x64 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Release|Win32.ActiveCfg = Release|Win32 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Release|Win32.Build.0 = Release|Win32 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Release|x64.ActiveCfg = Release|x64 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Release|x64.Build.0 = Release|x64 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Debug|Win32.ActiveCfg = Debug|Win32 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Debug|Win32.Build.0 = Debug|Win32 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Debug|x64.ActiveCfg = Release|x64 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Debug|x64.Build.0 = Release|x64 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Release|Win32.ActiveCfg = Release|Win32 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Release|Win32.Build.0 = Release|Win32 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Release|x64.ActiveCfg = Release|x64 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3} = {B04B8A8D-9F64-4F00-A01D-BE3392588862} {3E5CE550-982F-42A4-B631-0C65582F1EAF} = {B04B8A8D-9F64-4F00-A01D-BE3392588862} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {3D56059B-87AA-45B6-B008-6705531314F5} EndGlobalSection EndGlobal kanzi-cpp-2.6.0/msvc/Kanzi_VS2022.vcxproj000066400000000000000000000407061525576310500200120ustar00rootroot00000000000000īģŋ Debug Win32 Debug x64 Release Win32 Release x64 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3} Kanzi 10.0 Kanzi Application v143 MultiByte true Application v143 MultiByte true Application v143 MultiByte Application v143 MultiByte <_ProjectFileVersion>14.0.25431.1 $(SolutionDir)$(Configuration)\ $(Configuration)\ Kanzi64 NativeMinimumRules.ruleset $(SolutionDir)$(Configuration)\ $(Configuration)\ Kanzi64 NativeRecommendedRules.ruleset true Disabled Default false Neither false false _CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) Default MultiThreadedDebugDLL Level3 EditAndContinue true MachineX64 Disabled AnySuitable false Neither false false _CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) Default MultiThreadedDebugDLL Level3 ProgramDatabase stdcpp17 /Zc:__cplusplus AdvancedVectorExtensions true true MaxSpeed AnySuitable true _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreadedDLL true Level3 None Speed true Fast StreamingSIMDExtensions CompileAsCpp true true true MachineX86 MaxSpeed AnySuitable true _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreaded true Level3 ProgramDatabase Speed true Fast true CompileAsCpp AdvancedVectorExtensions stdcpp17 false /Zc:__cplusplus %(AdditionalOptions) Default true true true true false kanzi-cpp-2.6.0/msvc/Kanzi_VS2026.sln000066400000000000000000000050251525576310500171120ustar00rootroot00000000000000Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 18 VisualStudioVersion = 18.3.11520.95 MinimumVisualStudioVersion = 10.0.40219.1 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{B04B8A8D-9F64-4F00-A01D-BE3392588862}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Kanzi", "Kanzi_VS2026.vcxproj", "{A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "Libkanzi", "libkanzi_VS2026.vcxproj", "{3E5CE550-982F-42A4-B631-0C65582F1EAF}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 Debug|x64 = Debug|x64 Release|Win32 = Release|Win32 Release|x64 = Release|x64 EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Debug|Win32.ActiveCfg = Debug|Win32 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Debug|Win32.Build.0 = Debug|Win32 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Debug|x64.ActiveCfg = Debug|x64 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Debug|x64.Build.0 = Debug|x64 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Release|Win32.ActiveCfg = Release|Win32 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Release|Win32.Build.0 = Release|Win32 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Release|x64.ActiveCfg = Release|x64 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3}.Release|x64.Build.0 = Release|x64 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Debug|Win32.ActiveCfg = Debug|Win32 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Debug|Win32.Build.0 = Debug|Win32 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Debug|x64.ActiveCfg = Debug|x64 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Debug|x64.Build.0 = Debug|x64 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Release|Win32.ActiveCfg = Release|Win32 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Release|Win32.Build.0 = Release|Win32 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Release|x64.ActiveCfg = Release|x64 {3E5CE550-982F-42A4-B631-0C65582F1EAF}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE EndGlobalSection GlobalSection(NestedProjects) = preSolution {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3} = {B04B8A8D-9F64-4F00-A01D-BE3392588862} {3E5CE550-982F-42A4-B631-0C65582F1EAF} = {B04B8A8D-9F64-4F00-A01D-BE3392588862} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {2A8F4CEB-D2CA-4600-8D9A-9CDE16FB8726} EndGlobalSection EndGlobal kanzi-cpp-2.6.0/msvc/Kanzi_VS2026.vcxproj000066400000000000000000000461711525576310500200200ustar00rootroot00000000000000īģŋ Debug Win32 Debug x64 Release Win32 Release x64 {A7FB6DB8-D93B-4287-8ED8-0E39A52509B3} Kanzi 10.0 Kanzi Application v145 MultiByte true Application v145 MultiByte true Application v145 MultiByte Application v145 MultiByte $(ProjectDir)..\src <_ProjectFileVersion>14.0.25431.1 $(SolutionDir)$(Configuration)\ $(Configuration)\ Kanzi64 NativeMinimumRules.ruleset $(SolutionDir)$(Configuration)\ $(Configuration)\ Kanzi64 NativeRecommendedRules.ruleset true Disabled Default false Neither false false _CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) Default MultiThreadedDebugDLL Level3 EditAndContinue true MachineX64 Disabled AnySuitable false Neither false false _CRT_SECURE_NO_WARNINGS;_SCL_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) Default MultiThreadedDebugDLL Level3 ProgramDatabase stdcpp17 /Zc:__cplusplus AdvancedVectorExtensions true true MaxSpeed AnySuitable true _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreadedDLL true Level3 None Speed true Fast StreamingSIMDExtensions CompileAsCpp true true true MachineX86 MaxSpeed AnySuitable true _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreaded true Level3 ProgramDatabase Speed true Fast true CompileAsCpp AdvancedVectorExtensions stdcpp17 false /Zc:__cplusplus %(AdditionalOptions) Default true true true true $(KanziSourceRoot);%(AdditionalIncludeDirectories) false kanzi-cpp-2.6.0/msvc/README.md000066400000000000000000000002571525576310500156570ustar00rootroot00000000000000The Visual Studio 2026 solution and projects can be opened directly from this directory. For older Visual Studio versions, copy the corresponding files to `..\src` as before. kanzi-cpp-2.6.0/msvc/libkanzi_VS2022.vcxproj000066400000000000000000000332611525576310500205370ustar00rootroot00000000000000 Debug Win32 Release Win32 Debug x64 Release x64 15.0 {3E5CE550-982F-42A4-B631-0C65582F1EAF} libkanzi 10.0 Application true v141 MultiByte Application false v141 true MultiByte StaticLibrary true v141 MultiByte StaticLibrary false v143 true MultiByte .lib .lib Level3 Disabled true true _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreadedDebug Console Level3 Disabled true true Console Level3 MaxSpeed true true true true Console true true Level3 MaxSpeed true true true true AnySuitable Speed true MultiThreaded _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) stdcpp17 /Zc:__cplusplus %(AdditionalOptions) Console true true kanzi-cpp-2.6.0/msvc/libkanzi_VS2026.vcxproj000066400000000000000000000362501525576310500205440ustar00rootroot00000000000000 Debug Win32 Release Win32 Debug x64 Release x64 18.0 {3E5CE550-982F-42A4-B631-0C65582F1EAF} libkanzi 10.0 Application true v145 MultiByte Application false v145 true MultiByte StaticLibrary true v145 MultiByte StaticLibrary false v145 true MultiByte $(ProjectDir)..\src .lib .lib Level3 Disabled true true _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) MultiThreadedDebug Console Level3 Disabled true true Console Level3 MaxSpeed true true true true Console true true Level3 MaxSpeed true true true true AnySuitable Speed true MultiThreaded _CRT_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) stdcpp17 /Zc:__cplusplus %(AdditionalOptions) Console true true $(KanziSourceRoot);%(AdditionalIncludeDirectories) kanzi-cpp-2.6.0/src/000077500000000000000000000000001525576310500142135ustar00rootroot00000000000000kanzi-cpp-2.6.0/src/BitStreamException.hpp000066400000000000000000000030121525576310500204710ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_BitStreamException #define knz_BitStreamException #include #include #include "types.hpp" namespace kanzi { class BitStreamException : public std::runtime_error { private: int _code; public: enum BitStreamStatus { UNDEFINED = 0, INPUT_OUTPUT = 1, END_OF_STREAM = 2, INVALID_STREAM = 3, STREAM_CLOSED = 4 }; BitStreamException(const std::string& msg) : std::runtime_error(msg) { _code = UNDEFINED; } BitStreamException(const std::string& msg, int code) : std::runtime_error(msg), _code(code) { } #if __cplusplus >= 201103L BitStreamException(const BitStreamException&) = default; BitStreamException& operator=(const BitStreamException&) = default; #endif int error() const { return _code; } ~BitStreamException() NOEXCEPT {} }; } #endif kanzi-cpp-2.6.0/src/Context.hpp000066400000000000000000000104661525576310500163570ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Context #define knz_Context #include #include #if __cplusplus >= 201703L #include #endif #include "concurrent.hpp" // definition of CONCURRENCY_ENABLED namespace kanzi { #if __cplusplus >= 201703L // C++17+ version using std::variant typedef std::variant ContextVal; #else // C++98 / C++03 / C++11 / C++14 struct ContextVal { int64 lVal; std::string sVal; bool isString; ContextVal() : lVal(0), isString(false) {} ContextVal(int64 v) : lVal(v), isString(false) {} ContextVal(const std::string& s) : lVal(0), sVal(s), isString(true) {} }; #endif class Context { public: #ifdef CONCURRENCY_ENABLED Context(ThreadPool* p = nullptr) : _pool(p) {} Context(const Context& c) : _map(c._map), _pool(c._pool) {} Context(const Context& c, ThreadPool* p) : _map(c._map), _pool(p) {} Context& operator=(const Context& c) = default; #else Context() {} Context(const Context& c) : _map(c._map) {} Context& operator=(const Context& c) { _map = c._map; return *this; } #endif ~Context() {} bool has(const std::string& key) const; int getInt(const std::string& key, int defValue = 0) const; int64 getLong(const std::string& key, int64 defValue = 0) const; std::string getString(const std::string& key, const std::string& defValue = "") const; void putInt(const std::string& key, int value); void putLong(const std::string& key, int64 value); void putString(const std::string& key, const std::string& value); #ifdef CONCURRENCY_ENABLED ThreadPool* getPool() const { return _pool; } #endif private: std::map _map; #ifdef CONCURRENCY_ENABLED ThreadPool* _pool; #endif }; inline bool Context::has(const std::string& key) const { return _map.find(key) != _map.end(); } inline int Context::getInt(const std::string& key, int defValue) const { return int(getLong(key, defValue)); } inline int64 Context::getLong(const std::string& key, int64 defValue) const { const std::map::const_iterator it = _map.find(key); if (it == _map.end()) return defValue; #if __cplusplus >= 201703L if (std::holds_alternative(it->second)) return std::get(it->second); return defValue; #else return it->second.isString ? defValue : it->second.lVal; #endif } inline std::string Context::getString(const std::string& key, const std::string& defValue) const { const std::map::const_iterator it = _map.find(key); if (it == _map.end()) return defValue; #if __cplusplus >= 201703L if (std::holds_alternative(it->second)) return std::get(it->second); return defValue; #else return it->second.isString ? it->second.sVal : defValue; #endif } inline void Context::putInt(const std::string& key, int value) { #if __cplusplus >= 201703L _map[key] = int64(value); #else _map[key] = ContextVal((int64)value); #endif } inline void Context::putLong(const std::string& key, int64 value) { #if __cplusplus >= 201703L _map[key] = value; #else _map[key] = ContextVal(value); #endif } inline void Context::putString(const std::string& key, const std::string& value) { #if __cplusplus >= 201703L _map[key] = value; #else _map[key] = ContextVal(value); #endif } } #endif kanzi-cpp-2.6.0/src/EntropyDecoder.hpp000066400000000000000000000024421525576310500176540ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_EntropyDecoder #define knz_EntropyDecoder #include "InputBitStream.hpp" namespace kanzi { // EntropyDecoder entropy decodes data from a bitstream class EntropyDecoder { public: // Decode the array provided from the bitstream. Return the number of bytes // read from the bitstream virtual int decode(byte block[], uint blkptr, uint len) = 0; // Return the underlying bitstream virtual InputBitStream& getBitStream() const = 0; // Must be called before getting rid of the entropy decoder. // Trying to decode after a call to dispose gives undefined behavior virtual void dispose() = 0; virtual ~EntropyDecoder(){} }; } #endif kanzi-cpp-2.6.0/src/EntropyEncoder.hpp000066400000000000000000000024511525576310500176660ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_EntropyEncoder #define knz_EntropyEncoder #include "OutputBitStream.hpp" namespace kanzi { // EntropyEncoder entropy encodes data to a bitstream class EntropyEncoder { public: // Encode the array provided into the bitstream. Return the number of bytes // written to the bitstream virtual int encode(const byte block[], uint blkptr, uint len) = 0; // Return the underlying bitstream virtual OutputBitStream& getBitStream() const = 0; // Must be called before getting rid of the entropy encoder. // Trying to encode after a call to dispose gives undefined behavior virtual void dispose() = 0; virtual ~EntropyEncoder(){} }; } #endif kanzi-cpp-2.6.0/src/Error.hpp000066400000000000000000000026451525576310500160240ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Error #define knz_Error namespace kanzi { struct Error { public: enum ErrorCode { ERR_MISSING_PARAM = 1, ERR_BLOCK_SIZE = 2, ERR_INVALID_CODEC = 3, ERR_CREATE_COMPRESSOR = 4, ERR_CREATE_DECOMPRESSOR = 5, ERR_OUTPUT_IS_DIR = 6, ERR_OVERWRITE_FILE = 7, ERR_CREATE_FILE = 8, ERR_CREATE_BITSTREAM = 9, ERR_OPEN_FILE = 10, ERR_READ_FILE = 11, ERR_WRITE_FILE = 12, ERR_PROCESS_BLOCK = 13, ERR_CREATE_CODEC = 14, ERR_INVALID_FILE = 15, ERR_STREAM_VERSION = 16, ERR_CREATE_STREAM = 17, ERR_INVALID_PARAM = 18, ERR_CRC_CHECK = 19, ERR_RESERVED_NAME = 20, ERR_UNKNOWN = 127 }; }; } #endif kanzi-cpp-2.6.0/src/Event.cpp000066400000000000000000000160111525576310500157770ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include "Event.hpp" #include "util/strings.hpp" using namespace kanzi; Event::Event(Event::Type type, int id, const std::string& msg, WallTimer::TimeData evtTime) : _type(type) , _time(evtTime) , _msg(msg) , _id(id) , _size(0) , _offset(-1) , _hash(0) , _hashType(NO_HASH) , _skipFlags(0) , _info(nullptr) { } Event::Event(Event::Type type, int id, const HeaderInfo& info, WallTimer::TimeData evtTime) : _type(type) , _time(evtTime) , _msg("") , _id(id) , _size(0) , _offset(-1) , _hash(0) , _hashType(NO_HASH) , _skipFlags(0) { _info = new HeaderInfo(); _info->inputName = info.inputName; _info->bsVersion = info.bsVersion; _info->checksumSize = info.checksumSize; _info->blockSize = info.blockSize; _info->entropyType = info.entropyType; _info->transformType = info.transformType; _info->originalSize = info.originalSize; _info->fileSize = info.fileSize; } Event::Event(Event::Type type, int id, int64 size, WallTimer::TimeData evtTime, uint64 hash, HashType hashType, int64 offset, uint8 skipFlags) : _type(type) , _time(evtTime) , _msg() , _id(id) , _size(size) , _offset(offset) , _hash(hash) , _hashType(hashType) , _skipFlags(skipFlags) , _info(nullptr) { } Event::Event(const Event& other) : _type(other._type) , _time(other._time) , _msg(other._msg) , _id(other._id) , _size(other._size) , _offset(other._offset) , _hash(other._hash) , _hashType(other._hashType) , _skipFlags(other._skipFlags) , _info(nullptr) { if (other._info != nullptr) { _info = new HeaderInfo(); _info->inputName = other._info->inputName; _info->bsVersion = other._info->bsVersion; _info->checksumSize = other._info->checksumSize; _info->blockSize = other._info->blockSize; _info->entropyType = other._info->entropyType; _info->transformType = other._info->transformType; _info->originalSize = other._info->originalSize; _info->fileSize = other._info->fileSize; } } Event& Event::operator=(const Event& other) { if (this != &other) { _type = other._type; _time = other._time; _msg = other._msg; _id = other._id; _size = other._size; _offset = other._offset; _hash = other._hash; _hashType = other._hashType; _skipFlags = other._skipFlags; if (_info != nullptr) { delete _info; _info = nullptr; } if (other._info != nullptr) { _info = new HeaderInfo(); _info->inputName = other._info->inputName; _info->bsVersion = other._info->bsVersion; _info->checksumSize = other._info->checksumSize; _info->blockSize = other._info->blockSize; _info->entropyType = other._info->entropyType; _info->transformType = other._info->transformType; _info->originalSize = other._info->originalSize; _info->fileSize = other._info->fileSize; } } return *this; } #if defined(__cplusplus) && (__cplusplus >= 201103L) Event::Event(Event&& other) noexcept : _type(other._type) , _time(other._time) , _msg(std::move(other._msg)) , _id(other._id) , _size(other._size) , _offset(other._offset) , _hash(other._hash) , _hashType(other._hashType) , _skipFlags(other._skipFlags) , _info(other._info) { other._info = nullptr; } Event& Event::operator=(Event&& other) noexcept { if (this != &other) { _type = other._type; _time = other._time; _msg = std::move(other._msg); _id = other._id; _size = other._size; _offset = other._offset; _hash = other._hash; _hashType = other._hashType; _skipFlags = other._skipFlags; if (_info != nullptr) delete _info; _info = other._info; other._info = nullptr; } return *this; } #endif std::string Event::toString() const { if (_msg != "") return _msg; std::stringstream ss; ss << "{ \"type\":\"" << getTypeAsString() << "\""; if (_id >= 0) ss << ", \"id\":" << getId(); if (_info != nullptr) { ss << ", \"inputName\":\"" << escapeJSONString(_info->inputName) << "\""; ss << ", \"bsVersion\":" << _info->bsVersion; ss << ", \"checksum\":" << _info->checksumSize; ss << ", \"blockSize\":" << _info->blockSize; ss << ", \"entropy\":\"" << _info->entropyType << "\""; ss << ", \"transform\":\"" << _info->transformType << "\""; if (_info->fileSize >= 0) ss << ", \"compressed\":" << _info->fileSize; if (_info->originalSize >= 0) ss << ", \"original\":" << _info->originalSize; } else { ss << ", \"size\":" << getSize(); if (getType() != BLOCK_INFO) ss << ", \"time\":" << getTime().to_ms(); if (_hashType != NO_HASH) { ss << ", \"hash\":\""; ss << std::uppercase << std::setfill('0'); if (_hashType == SIZE_32) ss << std::setw(8) << std::hex << getHash() << "\""; else ss << std::setw(16) << std::hex << getHash() << "\""; ss << std::dec; } if (getType() == BLOCK_INFO) { ss << ", \"offset\":" << getOffset(); ss << ", \"skipFlags\":"; for (int i = 128; i >= 1; i >>= 1) ss << ((_skipFlags & i) == 0 ? "0" : "1"); } } ss << " }"; return ss.str(); } std::string Event::getTypeAsString() const { switch (_type) { case AFTER_HEADER_DECODING: return "AFTER_HEADER_DECODING"; case COMPRESSION_END: return "COMPRESSION_END"; case BEFORE_TRANSFORM: return "BEFORE_TRANSFORM"; case AFTER_TRANSFORM: return "AFTER_TRANSFORM"; case BEFORE_ENTROPY: return "BEFORE_ENTROPY"; case AFTER_ENTROPY: return "AFTER_ENTROPY"; case DECOMPRESSION_START: return "DECOMPRESSION_START"; case DECOMPRESSION_END: return "DECOMPRESSION_END"; case COMPRESSION_START: return "COMPRESSION_START"; case BLOCK_INFO: return "BLOCK_INFO"; default: return "Unknown Type"; } } kanzi-cpp-2.6.0/src/Event.hpp000066400000000000000000000060161525576310500160100ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Event #define knz_Event #include #include #include "types.hpp" #include "util/WallTimer.hpp" namespace kanzi { class Event { public: enum Type { COMPRESSION_START, COMPRESSION_END, BEFORE_TRANSFORM, AFTER_TRANSFORM, BEFORE_ENTROPY, AFTER_ENTROPY, DECOMPRESSION_START, DECOMPRESSION_END, AFTER_HEADER_DECODING, BLOCK_INFO }; enum HashType { NO_HASH, SIZE_32, SIZE_64 }; typedef struct HeaderInfo { std::string inputName; int bsVersion; int checksumSize; int blockSize; std::string entropyType; std::string transformType; int64 originalSize; int64 fileSize; } HeaderInfo; Event(Type type, int id, const std::string& msg, WallTimer::TimeData evtTime); Event(Type type, int id, int64 size, WallTimer::TimeData evtTime, uint64 hash = 0, HashType hashType = NO_HASH, int64 offset = -1, uint8 skipFlags = 0); Event(Type type, int id, const HeaderInfo& info, WallTimer::TimeData evtTime); Event(const Event& other); Event& operator=(const Event& other); #if defined(__cplusplus) && (__cplusplus >= 201103L) Event(Event&& other) noexcept; Event& operator=(Event&& other) noexcept; #endif virtual ~Event() { if (_info != nullptr) delete _info; } int getId() const { return _id; } int64 getSize() const { return _size; } Event::Type getType() const { return _type; } WallTimer::TimeData getTime() const { return _time; } uint64 getHash() const { return _hashType != NO_HASH ? _hash : 0; } int64 getOffset() const { return _offset; } HashType getHashType() const { return _hashType; } HeaderInfo* getInfo() const { return _info; } std::string toString() const; std::string getTypeAsString() const; private: Event::Type _type; WallTimer::TimeData _time; std::string _msg; int _id; int64 _size; int64 _offset; uint64 _hash; HashType _hashType; uint8 _skipFlags; HeaderInfo* _info; }; } #endif kanzi-cpp-2.6.0/src/Global.cpp000066400000000000000000000325061525576310500161250ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include "Global.hpp" #include "util/strings.hpp" using namespace kanzi; using namespace std; // int(Math.log2(x-1)) const int Global::LOG2[256] = { 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 8, }; // 4096*Math.log2(x) const int Global::LOG2_4096[257] = { 0, 0, 4096, 6492, 8192, 9511, 10588, 11499, 12288, 12984, 13607, 14170, 14684, 15157, 15595, 16003, 16384, 16742, 17080, 17400, 17703, 17991, 18266, 18529, 18780, 19021, 19253, 19476, 19691, 19898, 20099, 20292, 20480, 20662, 20838, 21010, 21176, 21338, 21496, 21649, 21799, 21945, 22087, 22226, 22362, 22495, 22625, 22752, 22876, 22998, 23117, 23234, 23349, 23462, 23572, 23680, 23787, 23892, 23994, 24095, 24195, 24292, 24388, 24483, 24576, 24668, 24758, 24847, 24934, 25021, 25106, 25189, 25272, 25354, 25434, 25513, 25592, 25669, 25745, 25820, 25895, 25968, 26041, 26112, 26183, 26253, 26322, 26390, 26458, 26525, 26591, 26656, 26721, 26784, 26848, 26910, 26972, 27033, 27094, 27154, 27213, 27272, 27330, 27388, 27445, 27502, 27558, 27613, 27668, 27722, 27776, 27830, 27883, 27935, 27988, 28039, 28090, 28141, 28191, 28241, 28291, 28340, 28388, 28437, 28484, 28532, 28579, 28626, 28672, 28718, 28764, 28809, 28854, 28898, 28943, 28987, 29030, 29074, 29117, 29159, 29202, 29244, 29285, 29327, 29368, 29409, 29450, 29490, 29530, 29570, 29609, 29649, 29688, 29726, 29765, 29803, 29841, 29879, 29916, 29954, 29991, 30027, 30064, 30100, 30137, 30172, 30208, 30244, 30279, 30314, 30349, 30384, 30418, 30452, 30486, 30520, 30554, 30587, 30621, 30654, 30687, 30719, 30752, 30784, 30817, 30849, 30880, 30912, 30944, 30975, 31006, 31037, 31068, 31099, 31129, 31160, 31190, 31220, 31250, 31280, 31309, 31339, 31368, 31397, 31426, 31455, 31484, 31513, 31541, 31569, 31598, 31626, 31654, 31681, 31709, 31737, 31764, 31791, 31818, 31846, 31872, 31899, 31926, 31952, 31979, 32005, 32031, 32058, 32084, 32109, 32135, 32161, 32186, 32212, 32237, 32262, 32287, 32312, 32337, 32362, 32387, 32411, 32436, 32460, 32484, 32508, 32533, 32557, 32580, 32604, 32628, 32651, 32675, 32698, 32722, 32745, 32768 }; char Global::BASE64_SYMBOLS[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; char Global::NUMERIC_SYMBOLS[] = "0123456789+-*/=,.:; "; char Global::DNA_SYMBOLS[] = "acgntuACGNTU"; // either T or U and N for unknown const Global Global::_singleton; int Global::SQUASH[4096]; int Global::STRETCH[4096]; Global::Global() { // 65536 /(1 + exp(-alpha*x)) with alpha ~= 0.54 const int INV_EXP[33] = { 0, 8, 22, 47, 88, 160, 283, 492, 848, 1451, 2459, 4117, 6766, 10819, 16608, 24127, 32768, 41409, 48928, 54717, 58770, 61419, 63077, 64085, 64688, 65044, 65253, 65376, 65448, 65489, 65514, 65528, 65536 }; for (int x = 1; x < 4096; x++) { const int w = x & 127; const int y = x >> 7; SQUASH[x - 1] = (INV_EXP[y] * (128 - w) + INV_EXP[y + 1] * w) >> 11; } SQUASH[4095] = 4095; int n = 0; for (int x = -2047; x <= 2047; x++) { const int sq = squash(x); while (n <= sq) STRETCH[n++] = x; if (n >= 4096) break; } STRETCH[4095] = 2047; #if defined(WIN32) || defined(_WIN32) || defined(_WIN64) static const string reserved[27] = { "AUX", "COM0", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "COMš", "COM²", "COMÂŗ", "CON", "LPT0", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", "NUL", "PRN" }; for (int i = 0; i < 27; i++) _reservedNames.insert(reserved[i]); #endif } // Return 1024 * log2(x). Max error is around 0.1% int Global::log2_1024(uint32 x) { if (x == 0) throw std::invalid_argument("Cannot calculate log of a negative or null value"); if (x < 256) return (Global::LOG2_4096[x] + 2) >> 2; const int log = _log2(x); if ((x & (x - 1)) == 0) return log << 10; return ((log - 7) * 1024) + ((LOG2_4096[x >> (log - 7)] + 2) >> 2); } int Global::log2(uint32 x) { if (x == 0) throw std::invalid_argument("Cannot calculate log of a negative or null value"); return _log2(x); } int Global::log2(uint64 x) { if (x == 0) throw std::invalid_argument("Cannot calculate log of a negative or null value"); return _log2(x); } // If withTotal is true, the last spot in each frequencies order 0 array is for the total void Global::computeHistogram(const kanzi::byte block[], int length, uint freqs[], bool isOrder0, bool withTotal) { if (length <= 0) { if ((isOrder0 == true) && (withTotal == true)) freqs[256] = 0; return; } const uint8* const base = reinterpret_cast(block); const uint8* p = base; if (isOrder0 == true) { if (withTotal == true) freqs[256] = uint(length); uint f0[256] = { 0 }; uint f1[256] = { 0 }; uint f2[256] = { 0 }; uint f3[256] = { 0 }; const uint8* end16 = &base[length & -16]; uint64 q; while (p < end16) { memcpy(&q, &p[0], 8); f0[uint8(q>>56)]++; f1[uint8(q>>48)]++; f2[uint8(q>>40)]++; f3[uint8(q>>32)]++; f0[uint8(q>>24)]++; f1[uint8(q>>16)]++; f2[uint8(q>>8)]++; f3[uint8(q)]++; memcpy(&q, &p[8], 8); f0[uint8(q>>56)]++; f1[uint8(q>>48)]++; f2[uint8(q>>40)]++; f3[uint8(q>>32)]++; f0[uint8(q>>24)]++; f1[uint8(q>>16)]++; f2[uint8(q>>8)]++; f3[uint8(q)]++; p += 16; } const uint8* end = &base[length]; while (p < end) freqs[*p++]++; for (int i = 0; i < 256; i++) freqs[i] += (f0[i] + f1[i] + f2[i] + f3[i]); } else { // Order 1 const int quarter = length >> 2; int n0 = 0 * quarter; int n1 = 1 * quarter; int n2 = 2 * quarter; int n3 = 3 * quarter; if (withTotal == true) { if (length < 32) { uint prv = 0; for (int i = 0; i < length; i++) { freqs[prv + uint(p[i])]++; freqs[prv + 256]++; prv = 257 * uint(p[i]); } } else { uint prv0 = 0; uint prv1 = 257 * uint(p[n1 - 1]); uint prv2 = 257 * uint(p[n2 - 1]); uint prv3 = 257 * uint(p[n3 - 1]); for (; n0 < quarter; n0++, n1++, n2++, n3++) { const uint cur0 = uint(p[n0]); const uint cur1 = uint(p[n1]); const uint cur2 = uint(p[n2]); const uint cur3 = uint(p[n3]); freqs[prv0 + cur0]++; freqs[prv0 + 256]++; freqs[prv1 + cur1]++; freqs[prv1 + 256]++; freqs[prv2 + cur2]++; freqs[prv2 + 256]++; freqs[prv3 + cur3]++; freqs[prv3 + 256]++; prv0 = 257 * cur0; prv1 = 257 * cur1; prv2 = 257 * cur2; prv3 = 257 * cur3; } for (; n3 < length; n3++) { freqs[prv3 + uint(p[n3])]++; freqs[prv3 + 256]++; prv3 = 257 * uint(p[n3]); } } } else { // order 1, no total if (length < 32) { uint prv = 0; for (int i = 0; i < length; i++) { freqs[prv + uint(p[i])]++; prv = 256 * uint(p[i]); } } else { uint prv0 = 0; uint prv1 = 256 * uint(p[n1 - 1]); uint prv2 = 256 * uint(p[n2 - 1]); uint prv3 = 256 * uint(p[n3 - 1]); for (; n0 < quarter; n0++, n1++, n2++, n3++) { const uint cur0 = uint(p[n0]); const uint cur1 = uint(p[n1]); const uint cur2 = uint(p[n2]); const uint cur3 = uint(p[n3]); freqs[prv0 + cur0]++; freqs[prv1 + cur1]++; freqs[prv2 + cur2]++; freqs[prv3 + cur3]++; prv0 = cur0 << 8; prv1 = cur1 << 8; prv2 = cur2 << 8; prv3 = cur3 << 8; } for (; n3 < length; n3++) { freqs[prv3 + uint(p[n3])]++; prv3 = uint(p[n3]) << 8; } } } } } // Return the zero order entropy scaled to the [0..1024] range // Incoming array size must be 256 int Global::computeOrder0Entropy1024(int blockLen, const uint histo[]) { if (blockLen == 0) return 0; uint64 sum = 0; const int logLength1024 = Global::log2_1024(uint32(blockLen)); for (int i = 0; i < 256; i++) { if (histo[i] == 0) continue; sum += ((uint64(histo[i]) * uint64(logLength1024 - Global::log2_1024(histo[i]))) >> 3); } return int(sum / uint64(blockLen)); } // Return the order 1 entropy scaled to the [0..1024] range // Incoming array size must be 65536 int Global::computeOrder1Entropy1024(int blockLen, const uint histo[]) { if (blockLen == 0) return 0; uint64 sum = 0; for (int prev = 0; prev < 256; prev++) { const int base = 256 * prev; uint total = 0; for (int cur = 0; cur < 256; cur++) total += histo[base + cur]; if (total == 0) continue; const int logTotal = Global::log2_1024(total); for (int cur = 0; cur < 256; cur++) { const uint freq = histo[base + cur]; if (freq != 0) sum += (uint64(freq) * uint64(logTotal - Global::log2_1024(freq))) >> 3; } } return int(sum / uint64(blockLen)); } void Global::computeJobsPerTask(int jobsPerTask[], int jobs, int tasks) { if (jobs <= 0) throw std::invalid_argument("Invalid number of jobs provided"); if (tasks <= 0) throw std::invalid_argument("Invalid number of tasks provided"); int q = (jobs <= tasks) ? 1 : jobs / tasks; int r = (jobs <= tasks) ? 0 : jobs - q * tasks; for (int i = 0; i < tasks; i++) jobsPerTask[i] = q; int n = 0; while (r != 0) { jobsPerTask[n]++; r--; n++; } } Global::DataType Global::detectSimpleType(int count, const uint freqs0[]) { int sum = 0; for (int i = 0; i < 12; i++) sum += freqs0[int(DNA_SYMBOLS[i])]; if (sum > (count - count / 12)) return DNA; sum = 0; for (int i = 0; i < 20; i++) sum += freqs0[int(NUMERIC_SYMBOLS[i])]; if (sum == count) return NUMERIC; // Last symbol with padding '=' sum = (freqs0[0x3D] == 1) ? 1 : 0; for (int i = 0; i < 64; i++) sum += freqs0[int(BASE64_SYMBOLS[i])]; if (sum == count) return BASE64; sum = 0; for (int i = 0; i < 256; i += 8) { sum += (freqs0[i+0] > 0) ? 1 : 0; sum += (freqs0[i+1] > 0) ? 1 : 0; sum += (freqs0[i+2] > 0) ? 1 : 0; sum += (freqs0[i+3] > 0) ? 1 : 0; sum += (freqs0[i+4] > 0) ? 1 : 0; sum += (freqs0[i+5] > 0) ? 1 : 0; sum += (freqs0[i+6] > 0) ? 1 : 0; sum += (freqs0[i+7] > 0) ? 1 : 0; } if (sum == 256) return BIN; return (sum <= 4) ? SMALL_ALPHABET : UNDEFINED; } #if defined(WIN32) || defined(_WIN32) || defined(_WIN64) bool Global::isReservedName(string fileName) { transform(fileName.begin(), fileName.end(), fileName.begin(), safeToUpper); return _singleton._reservedNames.find(fileName) != _singleton._reservedNames.end(); } #else bool Global::isReservedName(string) { return false; } #endif kanzi-cpp-2.6.0/src/Global.hpp000066400000000000000000000110601525576310500161220ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Global #define knz_Global #include #include #include "types.hpp" namespace kanzi { class Global { public: enum DataType { UNDEFINED, TEXT, MULTIMEDIA, EXE, NUMERIC, BASE64, DNA, BIN, UTF8, SMALL_ALPHABET }; static int stretch(int d); // ln(x / (1 - x)) static int squash(int d); // 1 / (1 + e-x) (inverse of stretch) static int log2(uint32 x); // fast, integer rounded static int log2(uint64 x); // fast, integer rounded static int _log2(uint32 x); // same as log2 minus check on input value static int _log2(uint64 x); // same as log2 minus check on input value static int trailingZeros(uint32 x); static int trailingZeros(uint64 x); static int log2_1024(uint32 x); // slow, accurate to 1/1024th static void computeJobsPerTask(int jobsPerTask[], int jobs, int tasks); static int computeOrder0Entropy1024(int blockLen, const uint histo[]); static int computeOrder1Entropy1024(int blockLen, const uint histo[]); static void computeHistogram(const byte block[], int end, uint freqs[], bool isOrder0=true, bool withTotal=false); static DataType detectSimpleType(int count, const uint histo[]); static bool isReservedName(std::string fileName); private: Global(); ~Global() {} static const Global _singleton; static const int LOG2_4096[257]; // 4096*Math.log2(x) static const int LOG2[256]; // int(Math.log2(x-1)) static int STRETCH[4096]; static int SQUASH[4096]; static char BASE64_SYMBOLS[]; static char DNA_SYMBOLS[]; static char NUMERIC_SYMBOLS[]; std::set _reservedNames; }; // return p = 1/(1 + exp(-d)), d scaled by 8 bits, p scaled by 12 bits inline int Global::squash(int d) { if (d >= 2048) return 4095; return (d <= -2048) ? 0 : SQUASH[d + 2047]; } inline int Global::stretch(int d) { return STRETCH[d]; } // x cannot be 0 inline int Global::_log2(uint32 x) { #if defined(_MSC_VER) unsigned long res; _BitScanReverse(&res, x); return int(res); #elif defined(__GNUG__) || defined(__clang__) return 31 ^ __builtin_clz(x); #else int res = 0; if (x >= 1 << 16) { x >>= 16; res = 16; } if (x >= 1 << 8) { x >>= 8; res += 8; } return res + Global::LOG2[x - 1]; #endif } // x cannot be 0 inline int Global::_log2(uint64 x) { #if defined(_MSC_VER) && defined(_M_AMD64) unsigned long res; _BitScanReverse64(&res, x); return int(res); #elif defined(__GNUG__) || defined(__clang__) return 63 ^ __builtin_clzll(x); #else int res = 0; if (x >= uint64(1) << 32) { x >>= 32; res = 32; } if (x >= uint64(1) << 16) { x >>= 16; res += 16; } if (x >= uint64(1) << 8) { x >>= 8; res += 8; } return res + Global::LOG2[x - 1]; #endif } // x cannot be 0 inline int Global::trailingZeros(uint32 x) { #if defined(_MSC_VER) unsigned long res; _BitScanForward(&res, x); return int(res); #elif defined(__GNUG__) || defined(__clang__) return __builtin_ctz(x); #else return _log2(x & (~x + 1)); #endif } // x cannot be 0 inline int Global::trailingZeros(uint64 x) { #if defined(_MSC_VER) && defined(_M_AMD64) unsigned long res; _BitScanForward64(&res, x); return int(res); #elif defined(__GNUG__) || defined(__clang__) return __builtin_ctzll(x); #else return _log2(x & (~x + 1)); #endif } } #endif kanzi-cpp-2.6.0/src/InputBitStream.hpp000066400000000000000000000027411525576310500176420ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_InputBitStream #define knz_InputBitStream #include "types.hpp" namespace kanzi { class InputBitStream { public: // Returns 1 or 0 virtual int readBit() = 0; // Length is the number of bits in [1..64]. Return the bits read as a long // Throws if the stream is closed. virtual uint64 readBits(uint length) = 0; // Read bits and put them in the byte array. Length is the number of bits // Return the number of bits read. // Throws if the stream is closed. virtual uint readBits(byte bits[], uint length) = 0; virtual void close() = 0; // Number of bits read virtual uint64 read() const = 0; // Return false when the bitstream is closed or the End-Of-Stream has been reached virtual bool hasMoreToRead() = 0; InputBitStream(){} virtual ~InputBitStream(){} }; } #endif kanzi-cpp-2.6.0/src/InputStream.hpp000066400000000000000000000013551525576310500172030ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_InputStream #define knz_InputStream #include namespace kanzi { // Maps to istream typedef std::istream InputStream; } #endif kanzi-cpp-2.6.0/src/Listener.hpp000066400000000000000000000014771525576310500165220ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Listener #define knz_Listener namespace kanzi { template class Listener { public: Listener(){} virtual void processEvent(const T& evt) = 0; virtual ~Listener(){} }; } #endif kanzi-cpp-2.6.0/src/Magic.hpp000066400000000000000000000122011525576310500157400ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Magic #define knz_Magic #include "Memory.hpp" namespace kanzi { struct Magic { static const uint NO_MAGIC = 0; static const uint JPG_MAGIC = 0xFFD8FFE0; static const uint GIF_MAGIC = 0x47494638; static const uint PDF_MAGIC = 0x25504446; static const uint ZIP_MAGIC = 0x504B0304; // Works for jar & office docs static const uint LZMA_MAGIC = 0x377ABCAF; // Works for 7z 37 7A BC AF 27 1C static const uint RAR_MAGIC = 0x52617221; // 52 61 72 21 1A 07 00 static const uint PNG_MAGIC = 0x89504E47; static const uint ELF_MAGIC = 0x7F454C46; static const uint MAC_MAGIC32 = 0xFEEDFACE; static const uint MAC_CIGAM32 = 0xCEFAEDFE; static const uint MAC_MAGIC64 = 0xFEEDFACF; static const uint MAC_CIGAM64 = 0xCFFAEDFE; static const uint ZSTD_MAGIC = 0x28B52FFD; static const uint BROTLI_MAGIC = 0x81CFB2CE; static const uint RIFF_MAGIC = 0x52494646; // WAV, AVI, WEBP static const uint CAB_MAGIC = 0x4D534346; static const uint FLAC_MAGIC = 0x664C6143; static const uint XZ_MAGIC = 0xFD377A58; // FD 37 7A 58 5A 00 static const uint KNZ_MAGIC = 0x4B414E5A; static const uint BZIP2_MAGIC = 0x425A68; static const uint MP3_ID3_MAGIC = 0x494433; static const uint GZIP_MAGIC = 0x1F8B; static const uint BMP_MAGIC = 0x424D; static const uint WIN_MAGIC = 0x4D5A; static const uint PBM_MAGIC = 0x5034; // bin only static const uint PGM_MAGIC = 0x5035; // bin only static const uint PPM_MAGIC = 0x5036; // bin only static uint getType(const byte src[]); static bool isCompressed(uint magic); static bool isMultimedia(uint magic); static bool isExecutable(uint magic); }; // 4 bytes must be readable in src inline uint Magic::getType(const byte src[]) { static const uint KEYS32[18] = { GIF_MAGIC, PDF_MAGIC, ZIP_MAGIC, LZMA_MAGIC, PNG_MAGIC, ELF_MAGIC, MAC_MAGIC32, MAC_CIGAM32, MAC_MAGIC64, MAC_CIGAM64, ZSTD_MAGIC, BROTLI_MAGIC, CAB_MAGIC, RIFF_MAGIC, FLAC_MAGIC, XZ_MAGIC, KNZ_MAGIC, RAR_MAGIC }; static const uint KEYS16[3] = { GZIP_MAGIC, BMP_MAGIC, WIN_MAGIC }; const uint key = uint(BigEndian::readInt32(&src[0])); if ((key & ~0x0F) == JPG_MAGIC) return key; if (((key >> 8) == BZIP2_MAGIC) || ((key >> 8) == MP3_ID3_MAGIC)) return key >> 8; const int n = sizeof(KEYS32) / sizeof(uint); for (int i = 0; i < n; i++) { if (key == KEYS32[i]) return key; } const uint key16 = key >> 16; for (int i = 0; i < 3; i++) { if (key16 == KEYS16[i]) return key16; } if ((key16 == PBM_MAGIC) || (key16 == PGM_MAGIC) || (key16 == PPM_MAGIC)) { const uint subkey = (key >> 8) & 0xFF; if ((subkey == 0x07) || (subkey == 0x0A) || (subkey == 0x0D) || (subkey == 0x20)) return key16; } return NO_MAGIC; } inline bool Magic::isCompressed(uint magic) { switch (magic) { case JPG_MAGIC: case GIF_MAGIC: case PNG_MAGIC: //case RIFF_MAGIC: may or may not be case LZMA_MAGIC: case ZSTD_MAGIC: case BROTLI_MAGIC: case CAB_MAGIC: case ZIP_MAGIC: case GZIP_MAGIC: case BZIP2_MAGIC: case FLAC_MAGIC: case MP3_ID3_MAGIC: case XZ_MAGIC: case KNZ_MAGIC: case RAR_MAGIC: return true; default: return false; } } inline bool Magic::isMultimedia(uint magic) { switch (magic) { case JPG_MAGIC: case GIF_MAGIC: case PNG_MAGIC: case RIFF_MAGIC: case FLAC_MAGIC: case MP3_ID3_MAGIC: case BMP_MAGIC: case PBM_MAGIC: case PGM_MAGIC: case PPM_MAGIC: return true; default: return false; } } inline bool Magic::isExecutable(uint magic) { switch (magic) { case ELF_MAGIC: case WIN_MAGIC: case MAC_MAGIC32: case MAC_CIGAM32: case MAC_MAGIC64: case MAC_CIGAM64: return true; default: return false; } } } #endif kanzi-cpp-2.6.0/src/Makefile000066400000000000000000000253631525576310500156640ustar00rootroot00000000000000#CC=clang #CXX=clang++ APP=kanzi APP_STATIC=$(APP)_static APP_DYNAMIC=$(APP)_dynamic ifeq ($(TCMALLOC_ENABLED), 1) LDFLAGS += -lpthread -ltcmalloc_minimal else LDFLAGS += -lpthread endif ifeq ($(CONCURRENCY_DISABLED), 1) CONCURRENCY_FLAG = -DCONCURRENCY_DISABLED endif ifndef USE_INTRINSICS USE_INTRINSICS = 0 endif ifeq ($(USE_INTRINSICS), 1) INTRINSICS_FLAG = -DUSE_INTRINSICS endif ifndef CXX_STD CXX_STD = c++17 endif OBJ_DIR_ROOT = obj OBJ_DIR_TAG = $(subst +,x,$(subst /,_,$(subst =,_,$(CXX_STD)))) OBJ_DIR = $(OBJ_DIR_ROOT)/$(OBJ_DIR_TAG) TEST_OBJ_DIR=$(OBJ_DIR)/test ifndef C_STD C_STD = c11 endif ifeq ($(OS),Windows_NT) DETECTED_OS := Windows else DETECTED_OS := $(shell uname -s) endif CFLAGS += -c -std=$(C_STD) -Wall -Wextra -O3 -fPIC -pedantic -march=native ifeq ($(DETECTED_OS),Windows) CXXFLAGS += -c -std=$(CXX_STD) -Wall -Wextra -O3 -fomit-frame-pointer -fPIC -DNDEBUG -pedantic -march=native -fno-rtti $(CONCURRENCY_FLAG) else ARCH ?= $(shell uname -m) CFLAGS += -D_FILE_OFFSET_BITS=64 CXXFLAGS += -D_FILE_OFFSET_BITS=64 # Check for both x86_64 (Linux) and amd64 (FreeBSD) ifneq ($(filter x86_64 amd64,$(ARCH)),) #CXXFLAGS += -c -std=$(CXX_STD) -fsanitize=undefined -ftrapv -D_FORTIFY_SOURCE=3 -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_FAST -fstrict-aliasing -Wall -Wextra -O3 -fomit-frame-pointer -fPIC -DNDEBUG -pedantic -march=native -fno-rtti $(CONCURRENCY_FLAG) CXXFLAGS += -c -std=$(CXX_STD) -fstrict-aliasing -Wall -Wextra -O3 -fomit-frame-pointer -fPIC -DNDEBUG -pedantic -march=native -fno-rtti $(CONCURRENCY_FLAG) else #CXXFLAGS += -c -std=$(CXX_STD) -fsanitize=signed-integer-overflow -ftrapv -D_FORTIFY_SOURCE=3 -Wall -Wextra -O3 -fPIC -DNDEBUG -pedantic -fno-rtti $(CONCURRENCY_FLAG) CXXFLAGS += -c -std=$(CXX_STD) -fstrict-aliasing -Wall -Wextra -Wpedantic -Wdeprecated -O3 -fPIC -DNDEBUG -pedantic -fno-rtti $(CONCURRENCY_FLAG) endif endif CXXFLAGS += $(INTRINSICS_FLAG) ifeq ($(DETECTED_OS),FreeBSD) STATIC_LINK_FLAGS := -static else ifeq ($(DETECTED_OS),Darwin) STATIC_LINK_FLAGS := else STATIC_LINK_FLAGS := -static-libstdc++ -static-libgcc endif LIB_COMMON_SOURCES=Global.cpp \ Event.cpp \ util/WallTimer.cpp \ entropy/EntropyUtils.cpp \ entropy/HuffmanCommon.cpp \ entropy/CMPredictor.cpp \ entropy/TPAQPredictor.cpp \ transform/AliasCodec.cpp \ transform/BWT.cpp \ transform/BWTS.cpp \ transform/DivSufSort.cpp \ transform/SBRT.cpp \ transform/BWTBlockCodec.cpp \ transform/LZCodec.cpp \ transform/FSDCodec.cpp \ transform/ROLZCodec.cpp \ transform/RLT.cpp \ transform/SRT.cpp \ transform/TextCodec.cpp \ transform/UTFCodec.cpp \ transform/EXECodec.cpp \ transform/ZRLT.cpp LIB_COMP_SOURCES=api/Compressor.cpp \ bitstream/DebugOutputBitStream.cpp \ bitstream/DefaultOutputBitStream.cpp \ io/CompressedOutputStream.cpp \ entropy/ANSRangeEncoder.cpp \ entropy/BinaryEntropyEncoder.cpp \ entropy/ExpGolombEncoder.cpp \ entropy/FPAQEncoder.cpp \ entropy/HuffmanEncoder.cpp \ entropy/RangeEncoder.cpp LIB_DECOMP_SOURCES=api/Decompressor.cpp \ bitstream/DebugInputBitStream.cpp \ bitstream/DefaultInputBitStream.cpp \ io/CompressedInputStream.cpp \ entropy/ANSRangeDecoder.cpp \ entropy/BinaryEntropyDecoder.cpp \ entropy/ExpGolombDecoder.cpp \ entropy/FPAQDecoder.cpp \ entropy/HuffmanDecoder.cpp \ entropy/RangeDecoder.cpp LIB_SOURCES=$(LIB_COMMON_SOURCES) $(LIB_COMP_SOURCES) $(LIB_DECOMP_SOURCES) # Define library object files LIB_OBJECTS=$(filter-out $(OBJ_DIR)/test/%.o,$(LIB_COMMON_OBJECTS) $(LIB_COMP_OBJECTS) $(LIB_DECOMP_OBJECTS)) TEST_SOURCES=test/TestEntropyCodec.cpp \ test/TestBWT.cpp \ test/TestCompressedStream.cpp \ test/TestDefaultBitStream.cpp \ test/TestFactories.cpp \ test/TestMalformedStream.cpp \ test/TestTransforms.cpp \ test/TestAPI.c APP_SOURCES=app/Kanzi.cpp \ app/InfoPrinter.cpp \ app/BlockCompressor.cpp \ app/BlockDecompressor.cpp SOURCES=$(LIB_SOURCES) $(APP_SOURCES) # Function to create object file paths, preserving the directory structure OBJ = $(OBJ_DIR)/$(patsubst %.cpp,%.o,$(1)) LIB_COMMON_OBJECTS=$(foreach src,$(LIB_COMMON_SOURCES),$(call OBJ,$(src))) LIB_COMP_OBJECTS=$(foreach src,$(LIB_COMP_SOURCES),$(call OBJ,$(src))) LIB_DECOMP_OBJECTS=$(foreach src,$(LIB_DECOMP_SOURCES),$(call OBJ,$(src))) LIB_OBJECTS=$(LIB_COMMON_OBJECTS) $(LIB_COMP_OBJECTS) $(LIB_DECOMP_OBJECTS) #TEST_OBJECTS=$(foreach src,$(TEST_SOURCES),$(call OBJ,$(src))) APP_OBJECTS=$(foreach src,$(APP_SOURCES),$(call OBJ,$(src))) OBJECTS=$(LIB_OBJECTS) $(APP_OBJECTS) $(TEST_OBJECTS) RPTS=$(SOURCES:.cpp=.optrpt) STATIC_LIB_SUFFIX := .a SHARED_LIB_SUFFIX := .so PROG_SUFFIX := # Default to -shared (Linux/GNU) SHARED_OPTION := -shared # Detect Windows or macOS ifeq ($(DETECTED_OS),Windows) STATIC_LIB_SUFFIX := .lib SHARED_LIB_SUFFIX := .dll PROG_SUFFIX := .exe SHARED_OPTION := -shared else ifeq ($(DETECTED_OS),Darwin) # macOS Specifics SHARED_LIB_SUFFIX := .dylib SHARED_OPTION := -dynamiclib endif STATIC_LIB := lib$(APP)$(STATIC_LIB_SUFFIX) SHARED_LIB := lib$(APP)$(SHARED_LIB_SUFFIX) STATIC_COMP_LIB := lib$(APP)comp$(STATIC_LIB_SUFFIX) STATIC_DECOMP_LIB := lib$(APP)decomp$(STATIC_LIB_SUFFIX) SHARED_COMP_LIB := lib$(APP)comp$(SHARED_LIB_SUFFIX) SHARED_DECOMP_LIB := lib$(APP)decomp$(SHARED_LIB_SUFFIX) all: lib test $(APP_STATIC) $(APP_DYNAMIC) libcomp: $(STATIC_COMP_LIB) $(SHARED_COMP_LIB) libdecomp: $(STATIC_DECOMP_LIB) $(SHARED_DECOMP_LIB) lib: $(STATIC_LIB) $(SHARED_LIB) # Create static libraries $(STATIC_LIB):$(LIB_OBJECTS) $(AR) cr ../lib/$@ $+ $(STATIC_COMP_LIB):$(LIB_COMP_OBJECTS) $(AR) cr ../lib/$@ $+ $(STATIC_DECOMP_LIB):$(LIB_DECOMP_OBJECTS) $(AR) cr ../lib/$@ $+ # Create shared libraries $(SHARED_LIB):$(LIB_OBJECTS) $(CXX) -o ../lib/$@ $(LDFLAGS) $(SHARED_OPTION) $+ $(SHARED_COMP_LIB):$(LIB_COMP_OBJECTS) $(CXX) -o ../lib/$@ $(LDFLAGS) $(SHARED_OPTION) $+ $(SHARED_DECOMP_LIB):$(LIB_DECOMP_OBJECTS) $(CXX) -o ../lib/$@ $(LDFLAGS) $(SHARED_OPTION) $+ # Targets for test executables testAPI: $(LIB_OBJECTS) $(TEST_OBJ_DIR)/TestAPI.o $(CXX) $^ -o ../bin/$@ $(LDFLAGS) testBWT: $(LIB_OBJECTS) $(TEST_OBJ_DIR)/TestBWT.o $(CXX) $^ -o ../bin/$@ $(LDFLAGS) testTransforms: $(LIB_OBJECTS) $(TEST_OBJ_DIR)/TestTransforms.o $(CXX) $^ -o ../bin/$@ $(LDFLAGS) testEntropyCodec: $(LIB_OBJECTS) $(TEST_OBJ_DIR)/TestEntropyCodec.o $(CXX) $^ -o ../bin/$@ $(LDFLAGS) testDefaultBitStream: $(LIB_OBJECTS) $(TEST_OBJ_DIR)/TestDefaultBitStream.o $(CXX) $^ -o ../bin/$@ $(LDFLAGS) testFactories: $(LIB_OBJECTS) $(TEST_OBJ_DIR)/TestFactories.o $(CXX) $^ -o ../bin/$@ $(LDFLAGS) testMalformedStream: $(LIB_OBJECTS) $(TEST_OBJ_DIR)/TestMalformedStream.o $(CXX) $^ -o ../bin/$@ $(LDFLAGS) testCompressedStream: $(LIB_OBJECTS) $(TEST_OBJ_DIR)/TestCompressedStream.o $(CXX) $^ -o ../bin/$@ $(LDFLAGS) test: testAPI testBWT testTransforms testEntropyCodec testDefaultBitStream testFactories testMalformedStream testCompressedStream # Default executable target kanzi: $(LIB_OBJECTS) $(APP_OBJECTS) $(CXX) $^ -o ../bin/$@ $(LDFLAGS) # Statically linked executable ifeq ($(DETECTED_OS),Darwin) # macOS does not support -static => disable static target kanzi_static: @echo "Static linking is not supported on macOS" else # Only add the flags if we aren't on macOS kanzi_static: LDFLAGS += $(STATIC_LINK_FLAGS) kanzi_static: $(LIB_OBJECTS) $(APP_OBJECTS) $(CXX) $^ -o ../bin/kanzi_static$(PROG_SUFFIX) $(LDFLAGS) endif # Dynamically linked executable kanzi_dynamic: LDFLAGS := $(filter-out -static,$(LDFLAGS)) kanzi_dynamic: $(LIB_OBJECTS) $(APP_OBJECTS) $(CXX) $^ -o ../bin/kanzi_dynamic$(PROG_SUFFIX) $(LDFLAGS) # Install / uninstall (may require sudo or admin rights) ifeq ($(DETECTED_OS),Windows) INSTALL_DIR=C:/Program\ Files else INSTALL_DIR=/usr/local MAN_DIR=$(INSTALL_DIR)/share/man/man1 endif print-vars: @echo "OS=$(DETECTED_OS)" @echo "APP=$(APP)" @echo "CXX=$(CXX)" @echo "CXXFLAGS=$(CXXFLAGS)" @echo "LDFLAGS=$(LDFLAGS)" @echo "INSTALL_DIR=$(INSTALL_DIR)" install: $(STATIC_LIB) $(SHARED_LIB) $(APP) ifeq ($(DETECTED_OS),Windows) copy /Y ..\bin\$(APP)$(PROG_SUFFIX) $(INSTALL_DIR) else install -d $(INSTALL_DIR)/lib install -m 644 ../lib/$(STATIC_LIB) $(INSTALL_DIR)/lib install -m 644 ../lib/$(SHARED_LIB) $(INSTALL_DIR)/lib install -d $(INSTALL_DIR)/include/kanzi install -m 644 ./*.hpp $(INSTALL_DIR)/include/kanzi install -d $(INSTALL_DIR)/include/kanzi/api install -m 644 ./api/*.hpp $(INSTALL_DIR)/include/kanzi/api install -d $(INSTALL_DIR)/include/kanzi/io install -m 644 ./io/*.hpp $(INSTALL_DIR)/include/kanzi/io install -d $(INSTALL_DIR)/include/kanzi/entropy install -m 644 ./entropy/*.hpp $(INSTALL_DIR)/include/kanzi/entropy install -d $(INSTALL_DIR)/include/kanzi/bitstream install -m 644 ./bitstream/*.hpp $(INSTALL_DIR)/include/kanzi/bitstream install -d $(INSTALL_DIR)/include/kanzi/transform install -m 644 ./transform/*.hpp $(INSTALL_DIR)/include/kanzi/transform install -d $(INSTALL_DIR)/include/kanzi/util install -m 644 ./util/*.hpp $(INSTALL_DIR)/include/kanzi/util install -d $(INSTALL_DIR)/bin install -m577 ../bin/$(APP)$(PROG_SUFFIX) $(INSTALL_DIR)/bin install -d $(MAN_DIR) if [ -f ../doc/kanzi.1.gz ]; then \ install -m 644 ../doc/kanzi.1.gz $(MAN_DIR)/$(APP).1.gz; \ elif [ -f ../doc/kanzi.1 ]; then \ gzip -n -c ../doc/kanzi.1 > $(MAN_DIR)/$(APP).1.gz; \ chmod 644 $(MAN_DIR)/$(APP).1.gz; \ else \ echo "Error: missing ../doc/kanzi.1.gz or ../doc/kanzi.1"; \ exit 1; \ fi endif # Uninstall may require sudo or admin rights uninstall: ifeq ($(DETECTED_OS),Windows) del /Q $(INSTALL_DIR)\$(APP)$(PROG_SUFFIX) else rm -f -r $(INSTALL_DIR)/include/$(APP) rm -f $(INSTALL_DIR)/lib/$(STATIC_LIB) rm -f $(INSTALL_DIR)/lib/$(SHARED_LIB) rm -f $(INSTALL_DIR)/bin/$(APP_STATIC)$(PROG_SUFFIX) rm -f $(INSTALL_DIR)/bin/$(APP_DYNAMIC)$(PROG_SUFFIX) rm -f $(INSTALL_DIR)/bin/$(APP)$(PROG_SUFFIX) rm -f $(MAN_DIR)/$(APP).1.gz endif clean: ifeq ($(DETECTED_OS),Windows) if exist $(OBJ_DIR_ROOT) rmdir /S /Q $(OBJ_DIR_ROOT) ..\bin\$(APP)$(PROG_SUFFIX) \ ..\bin\$(APP_STATIC)$(PROG_SUFFIX) \ ..\bin\$(APP_DYNAMIC)$(PROG_SUFFIX) ..\bin\test*$(PROG_SUFFIX) \ ..\lib\$(STATIC_LIB) ..\lib\$(SHARED_LIB) \ ..\lib\$(STATIC_COMP_LIB) ..\lib\$(STATIC_DECOMP_LIB) \ ..\lib\$(SHARED_COMP_LIB) ..\lib\$(SHARED_DECOMP_LIB) else rm -f ../bin/test*$(PROG_SUFFIX) $(OBJECTS) $(RPTS) \ ../bin/$(APP)$(PROG_SUFFIX) \ ../bin/$(APP_STATIC)$(PROG_SUFFIX) \ ../bin/$(APP_DYNAMIC)$(PROG_SUFFIX) \ ../lib/$(STATIC_LIB) ../lib/$(SHARED_LIB) \ ../lib/$(STATIC_COMP_LIB) ../lib/$(STATIC_DECOMP_LIB) \ ../lib/$(SHARED_COMP_LIB) ../lib/$(SHARED_DECOMP_LIB) rm -rf $(OBJ_DIR_ROOT) endif ifeq ($(DETECTED_OS),Windows) MKDIR = if not exist $(subst /,\,$(dir $@)) mkdir $(subst /,\,$(dir $@)) else MKDIR = mkdir -p $(dir $@) endif $(OBJ_DIR)/%.o: %.cpp @$(MKDIR) $(CXX) $(CXXFLAGS) $< -o $@ $(OBJ_DIR)/%.o: %.c @$(MKDIR) $(CC) $(CFLAGS) $< -o $@ kanzi-cpp-2.6.0/src/Memory.hpp000066400000000000000000000337611525576310500162060ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Memory #define knz_Memory #if __cplusplus >= 202002L #include #endif #include #include "types.hpp" #if defined(USE_INTRINSICS) #if defined(__ARM_NEON) || defined(__aarch64__) #include #elif defined(__AVX512F__) || defined(__AVX2__) || defined(__SSE2__) #include #endif #endif namespace kanzi { // Prefetch helpers static KANZI_ALWAYS_INLINE void prefetchRead(const void* ptr) { #if defined(__GNUG__) || defined(__clang__) __builtin_prefetch(ptr, 0, 1); #elif defined(__x86_64__) || defined(_M_AMD64) _mm_prefetch((const char*)ptr, _MM_HINT_T0); #elif defined(_M_ARM) __prefetch(ptr); #elif defined(_M_ARM64) __prefetch2(ptr, 1); #endif } static KANZI_ALWAYS_INLINE void prefetchWrite(const void* ptr) { #if defined(__GNUG__) || defined(__clang__) __builtin_prefetch(ptr, 1, 1); #elif defined(__x86_64__) || defined(_M_AMD64) _mm_prefetch((const char*)ptr, _MM_HINT_T0); #elif defined(_M_ARM) __prefetchw(ptr); #elif defined(_M_ARM64) __prefetch2(ptr, 17); #endif } // Byte-swap helpers static KANZI_ALWAYS_INLINE uint16 knz_bswap16(uint16 x) { #if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 5) return __builtin_bswap16(x); #elif defined(_MSC_VER) return _byteswap_ushort(x); #else return (uint16)((x >> 8) | (x << 8)); #endif } static KANZI_ALWAYS_INLINE uint32 knz_bswap32(uint32 x) { #if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 5) return __builtin_bswap32(x); #elif defined(_MSC_VER) return _byteswap_ulong(x); #else return ((x >> 24) | ((x >> 8) & 0xFF00) | ((x << 8) & 0xFF0000) | (x << 24)); #endif } static KANZI_ALWAYS_INLINE uint64 knz_bswap64(uint64 x) { #if defined(__clang__) || (defined(__GNUC__) && __GNUC__ >= 5) return __builtin_bswap64(x); #elif defined(_MSC_VER) return _byteswap_uint64(x); #else x = ((x & 0xFFFFFFFF00000000ull) >> 32) | ((x & 0xFFFFFFFFull) << 32); x = ((x & 0xFFFF0000FFFF0000ull) >> 16) | ((x & 0xFFFF0000FFFFull) << 16); x = ((x & 0xFF00FF00FF00FF00ull) >> 8) | ((x & 0xFF00FF00FF00FFull) << 8); return x; #endif } #if defined(USE_INTRINSICS) && (defined(__ARM_NEON) || defined(__aarch64__)) static KANZI_ALWAYS_INLINE bool memEq4(const byte* x, const byte* y) { const uint32x2_t a = vld1_dup_u32(reinterpret_cast(x)); const uint32x2_t b = vld1_dup_u32(reinterpret_cast(y)); return vget_lane_u32(vceq_u32(a, b), 0) != 0; } static KANZI_ALWAYS_INLINE bool memEq8(const byte* x, const byte* y) { #if defined(__aarch64__) const uint64x1_t a = vld1_u64(reinterpret_cast(x)); const uint64x1_t b = vld1_u64(reinterpret_cast(y)); return vget_lane_u64(vceq_u64(a, b), 0) != 0; #else const uint32x2_t a = vld1_u32(reinterpret_cast(x)); const uint32x2_t b = vld1_u32(reinterpret_cast(y)); const uint32x2_t eq = vceq_u32(a, b); return (vget_lane_u32(eq, 0) != 0) && (vget_lane_u32(eq, 1) != 0); #endif } static KANZI_ALWAYS_INLINE void memCp8(byte* dst, const byte* src) { vst1_u8(reinterpret_cast(dst), vld1_u8(reinterpret_cast(src))); } static KANZI_ALWAYS_INLINE void memCp16(byte* dst, const byte* src) { vst1q_u8(reinterpret_cast(dst), vld1q_u8(reinterpret_cast(src))); } #elif defined(USE_INTRINSICS) && defined(__AVX512F__) static KANZI_ALWAYS_INLINE bool memEq4(const byte* x, const byte* y) { const __mmask16 mask = 0x0001; const __m512i a = _mm512_maskz_loadu_epi32(mask, x); const __m512i b = _mm512_maskz_loadu_epi32(mask, y); return _mm512_mask_cmpeq_epi32_mask(mask, a, b) == mask; } static KANZI_ALWAYS_INLINE bool memEq8(const byte* x, const byte* y) { const __mmask8 mask = 0x01; const __m512i a = _mm512_maskz_loadu_epi64(mask, x); const __m512i b = _mm512_maskz_loadu_epi64(mask, y); return _mm512_mask_cmpeq_epi64_mask(mask, a, b) == mask; } static KANZI_ALWAYS_INLINE void memCp8(byte* dst, const byte* src) { const __m512i value = _mm512_maskz_loadu_epi64(0x01, src); _mm512_mask_storeu_epi64(dst, 0x01, value); } static KANZI_ALWAYS_INLINE void memCp16(byte* dst, const byte* src) { const __m128i value = _mm_loadu_si128(reinterpret_cast(src)); _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), value); } #elif defined(USE_INTRINSICS) && defined(__AVX2__) static KANZI_ALWAYS_INLINE bool memEq4(const byte* x, const byte* y) { const __m256i mask = _mm256_set_epi32(0, 0, 0, 0, 0, 0, 0, -1); const __m256i a = _mm256_maskload_epi32(reinterpret_cast(x), mask); const __m256i b = _mm256_maskload_epi32(reinterpret_cast(y), mask); return (_mm256_movemask_epi8(_mm256_cmpeq_epi32(a, b)) & 0x0F) == 0x0F; } static KANZI_ALWAYS_INLINE bool memEq8(const byte* x, const byte* y) { const __m256i mask = _mm256_set_epi64x(0, 0, 0, -1); const __m256i a = _mm256_maskload_epi64(reinterpret_cast(x), mask); const __m256i b = _mm256_maskload_epi64(reinterpret_cast(y), mask); return (_mm256_movemask_epi8(_mm256_cmpeq_epi64(a, b)) & 0xFF) == 0xFF; } static KANZI_ALWAYS_INLINE void memCp8(byte* dst, const byte* src) { const __m256i mask = _mm256_set_epi64x(0, 0, 0, -1); const __m256i value = _mm256_maskload_epi64(reinterpret_cast(src), mask); _mm256_maskstore_epi64(reinterpret_cast(dst), mask, value); } static KANZI_ALWAYS_INLINE void memCp16(byte* dst, const byte* src) { const __m128i value = _mm_loadu_si128(reinterpret_cast(src)); _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), value); } #elif defined(USE_INTRINSICS) && defined(__SSE2__) static KANZI_ALWAYS_INLINE bool memEq4(const byte* x, const byte* y) { const __m128i va = _mm_loadu_si32(x); const __m128i vb = _mm_loadu_si32(y); return _mm_cvtsi128_si32(_mm_cmpeq_epi32(va, vb)) != 0; } static KANZI_ALWAYS_INLINE bool memEq8(const byte* x, const byte* y) { const __m128i a = _mm_loadl_epi64(reinterpret_cast(x)); const __m128i b = _mm_loadl_epi64(reinterpret_cast(y)); return _mm_movemask_epi8(_mm_cmpeq_epi8(a, b)) == 0xFF; } static KANZI_ALWAYS_INLINE void memCp8(byte* dst, const byte* src) { _mm_storel_epi64(reinterpret_cast<__m128i*>(dst), _mm_loadl_epi64(reinterpret_cast(src))); } static KANZI_ALWAYS_INLINE void memCp16(byte* dst, const byte* src) { _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), _mm_loadu_si128(reinterpret_cast(src))); } #else static KANZI_ALWAYS_INLINE bool memEq4(const byte* x, const byte* y) { return std::memcmp(x, y, 4) == 0; } static KANZI_ALWAYS_INLINE bool memEq8(const byte* x, const byte* y) { return std::memcmp(x, y, 8) == 0; } static KANZI_ALWAYS_INLINE void memCp8(byte* dst, const byte* src) { memcpy(dst, src, 8); } static KANZI_ALWAYS_INLINE void memCp16(byte* dst, const byte* src) { memcpy(dst, src, 16); } #endif static KANZI_ALWAYS_INLINE void memXor8(byte* dst, const byte* x, const byte* y) { #if defined(USE_INTRINSICS) && (defined(__ARM_NEON) || defined(__aarch64__)) vst1_u8(reinterpret_cast(dst), veor_u8(vld1_u8(reinterpret_cast(x)), vld1_u8(reinterpret_cast(y)))); #elif defined(USE_INTRINSICS) && defined(__AVX512F__) const __mmask8 mask = 0x01; const __m512i a = _mm512_maskz_loadu_epi64(mask, x); const __m512i b = _mm512_maskz_loadu_epi64(mask, y); _mm512_mask_storeu_epi64(dst, mask, _mm512_xor_si512(a, b)); #elif defined(USE_INTRINSICS) && defined(__AVX2__) const __m256i mask = _mm256_set_epi64x(0, 0, 0, -1); const __m256i a = _mm256_maskload_epi64(reinterpret_cast(x), mask); const __m256i b = _mm256_maskload_epi64(reinterpret_cast(y), mask); _mm256_maskstore_epi64(reinterpret_cast(dst), mask, _mm256_xor_si256(a, b)); #elif defined(USE_INTRINSICS) && defined(__SSE2__) const __m128i a = _mm_loadl_epi64(reinterpret_cast(x)); const __m128i b = _mm_loadl_epi64(reinterpret_cast(y)); _mm_storel_epi64(reinterpret_cast<__m128i*>(dst), _mm_xor_si128(a, b)); #else uint64 a; uint64 b; memcpy(&a, x, sizeof(uint64)); memcpy(&b, y, sizeof(uint64)); a ^= b; memcpy(dst, &a, sizeof(uint64)); #endif } static KANZI_ALWAYS_INLINE void memXor16(byte* dst, const byte* x, const byte* y) { #if defined(USE_INTRINSICS) && (defined(__ARM_NEON) || defined(__aarch64__)) vst1q_u8(reinterpret_cast(dst), veorq_u8(vld1q_u8(reinterpret_cast(x)), vld1q_u8(reinterpret_cast(y)))); #elif defined(USE_INTRINSICS) && defined(__AVX512F__) const __mmask8 mask = 0x03; const __m512i a = _mm512_maskz_loadu_epi64(mask, x); const __m512i b = _mm512_maskz_loadu_epi64(mask, y); _mm512_mask_storeu_epi64(dst, mask, _mm512_xor_si512(a, b)); #elif defined(USE_INTRINSICS) && defined(__AVX2__) const __m256i mask = _mm256_set_epi64x(0, 0, -1, -1); const __m256i a = _mm256_maskload_epi64(reinterpret_cast(x), mask); const __m256i b = _mm256_maskload_epi64(reinterpret_cast(y), mask); _mm256_maskstore_epi64(reinterpret_cast(dst), mask, _mm256_xor_si256(a, b)); #elif defined(USE_INTRINSICS) && defined(__SSE2__) const __m128i a = _mm_loadu_si128(reinterpret_cast(x)); const __m128i b = _mm_loadu_si128(reinterpret_cast(y)); _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), _mm_xor_si128(a, b)); #else memXor8(dst, x, y); memXor8(dst + 8, x + 8, y + 8); #endif } #define KANZI_MEM_EQ4(x, y) (::kanzi::memEq4((x), (y))) #define KANZI_MEM_EQ8(x, y) (::kanzi::memEq8((x), (y))) #define KANZI_MEM_CP8(dst, src) (::kanzi::memCp8((dst), (src))) #define KANZI_MEM_CP16(dst, src) (::kanzi::memCp16((dst), (src))) #define KANZI_MEM_XOR8(dst, x, y) (::kanzi::memXor8((dst), (x), (y))) #define KANZI_MEM_XOR16(dst, x, y) (::kanzi::memXor16((dst), (x), (y))) // Detect host endianness #if __cplusplus >= 202002L static_assert(std::endian::native == std::endian::little || std::endian::native == std::endian::big, "Kanzi supports only little- and big-endian hosts"); #else #ifndef HOST_IS_LITTLE #if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) || defined(__BIG_ENDIAN__) #define HOST_IS_LITTLE 0 #else #define HOST_IS_LITTLE 1 #endif #endif #endif template static KANZI_ALWAYS_INLINE T readEndian(const byte* p) { T val; memcpy(&val, p, sizeof(T)); // Swap if host and source endianness differ #if __cplusplus >= 202002L if constexpr (SourceIsBigEndian != (std::endian::native == std::endian::big)) { #elif HOST_IS_LITTLE if (SourceIsBigEndian) { #else if (!SourceIsBigEndian) { #endif if (sizeof(T) == 2) val = (T)knz_bswap16((uint16)val); else if (sizeof(T) == 4) val = (T)knz_bswap32((uint32)val); else if (sizeof(T) == 8) val = (T)knz_bswap64((uint64)val); } return val; } template static KANZI_ALWAYS_INLINE void writeEndian(byte* p, T val) { #if __cplusplus >= 202002L if constexpr (TargetIsBigEndian != (std::endian::native == std::endian::big)) { #elif HOST_IS_LITTLE if (TargetIsBigEndian) { #else if (!TargetIsBigEndian) { #endif if (sizeof(T) == 2) val = (T)knz_bswap16((uint16)val); else if (sizeof(T) == 4) val = (T)knz_bswap32((uint32)val); else if (sizeof(T) == 8) val = (T)knz_bswap64((uint64)val); } memcpy(p, &val, sizeof(T)); } class BigEndian { public: static int64 readLong64(const byte* p) { return readEndian(p); } static int32 readInt32(const byte* p) { return readEndian(p); } static int16 readInt16(const byte* p) { return readEndian(p); } static void writeLong64(byte* p, int64 v) { writeEndian(p, v); } static void writeInt32(byte* p, int32 v) { writeEndian(p, v); } static void writeInt16(byte* p, int16 v) { writeEndian(p, v); } }; class LittleEndian { public: static int64 readLong64(const byte* p) { return readEndian(p); } static int32 readInt32(const byte* p) { return readEndian(p); } static int16 readInt16(const byte* p) { return readEndian(p); } static void writeLong64(byte* p, int64 v) { writeEndian(p, v); } static void writeInt32(byte* p, int32 v) { writeEndian(p, v); } static void writeInt16(byte* p, int16 v) { writeEndian(p, v); } }; } // namespace kanzi #endif kanzi-cpp-2.6.0/src/OutputBitStream.hpp000066400000000000000000000030721525576310500200410ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_OutputBitStream #define knz_OutputBitStream #include "types.hpp" namespace kanzi { class OutputBitStream { public: // Write the least significant bit of the input integer // Throws if the stream is closed. virtual void writeBit(int bit) = 0; // Length is the number of bits in [0..64]. A zero length is a successful // no-op regardless of the stream status. Return the number of bits written. // Throws if the stream is closed and length is not zero. virtual uint writeBits(uint64 bits, uint length) = 0; // Write bits ouf of the byte array. Length is the number of bits. // Return the number of bits written. // Throws if the stream is closed. virtual uint writeBits(const byte bits[], uint length) = 0; virtual void close() = 0; // Number of bits written virtual uint64 written() const = 0; OutputBitStream(){} virtual ~OutputBitStream(){} }; } #endif kanzi-cpp-2.6.0/src/OutputStream.hpp000066400000000000000000000013601525576310500174000ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_OutputStream #define knz_OutputStream #include namespace kanzi { // Maps to ostream typedef std::ostream OutputStream; } #endif kanzi-cpp-2.6.0/src/Predictor.hpp000066400000000000000000000022031525576310500166540ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Predictor #define knz_Predictor namespace kanzi { // Predictor predicts the probability of the next bit being 1. class Predictor { public: Predictor(){} // Updates the internal probability model based on the observed bit virtual void update(int bit) = 0; // Returns the value representing the probability of the next bit being 1 // in the [0..4095] range. // E.G. 410 represents roughly a probability of 10% for 1 virtual int get() = 0; virtual ~Predictor(){} }; } #endif kanzi-cpp-2.6.0/src/Seekable.hpp000066400000000000000000000016751525576310500164500ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Seekable #define knz_Seekable #include "types.hpp" namespace kanzi { class Seekable { public: Seekable(){} // return position in bits virtual int64 tell() = 0; // position in bits // return success or failure virtual bool seek(int64 position) = 0; virtual ~Seekable(){} }; } #endif kanzi-cpp-2.6.0/src/SliceArray.hpp000066400000000000000000000034551525576310500167710ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_SliceArray #define knz_SliceArray namespace kanzi { template class SliceArray { public: T* _array; int _length; // buffer length (a.k.a capacity) int _index; SliceArray(T* arr, int len, int index = 0) : _array(arr), _length(len), _index(index) {} #if __cplusplus < 201103L SliceArray(const SliceArray& sa) { _array = sa._array; _length = sa._length; _index = sa._index; } SliceArray& operator=(const SliceArray& sa); ~SliceArray(){} // does not deallocate buffer memory #else SliceArray(SliceArray&& sa) noexcept = default; SliceArray& operator=(SliceArray&& sa) noexcept = default; ~SliceArray() = default; #endif // Utility methods static bool isValid(const SliceArray& sa); }; template inline bool SliceArray::isValid(const SliceArray& sa) { return ((sa._array != nullptr) && (sa._index >= 0) && (sa._length >= 0) && (sa._index <= sa._length)); } #if __cplusplus < 201103L template inline SliceArray& SliceArray::operator=(const SliceArray& sa) { _array = sa._array; _length = sa._length; _index = sa._index; return *this; } #endif } #endif kanzi-cpp-2.6.0/src/Transform.hpp000066400000000000000000000034731525576310500167060ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Transform #define knz_Transform #include "SliceArray.hpp" namespace kanzi { // Transform is a class used to transform an input byte array and write // the result to an output byte array. The result may have a different size. // The transform must be stateless to ensure that the compression results // are the same regardless of the number of jobs (ie no information is retained // between two invocations of forward or inverse). template class Transform { public: Transform(){} // The index of each slice array returns the number of bytes read & written respectively. // If the method returns false, the values of the indexes are best effort only. virtual bool forward(SliceArray& src, SliceArray& dst, int length) = 0; // The index of each slice array returns the number of bytes read & written respectively. // If the method returns false, the values of the indexes are best effort only. virtual bool inverse(SliceArray& src, SliceArray& dst, int length) = 0; // Return the maximum possible length of the transformed data virtual int getMaxEncodedLength(int srcLen) const = 0; virtual ~Transform(){} }; } #endif kanzi-cpp-2.6.0/src/api/000077500000000000000000000000001525576310500147645ustar00rootroot00000000000000kanzi-cpp-2.6.0/src/api/Compressor.cpp000066400000000000000000000236131525576310500176310ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include "Compressor.hpp" #include "../types.hpp" #include "../Error.hpp" #include "../io/IOException.hpp" #include "../io/CompressedOutputStream.hpp" #include "../transform/TransformFactory.hpp" #include "../entropy/EntropyEncoderFactory.hpp" // Note stat64/lstat64 are deprecated on MacOS/Linux // Use _FILE_OFFSET_BITS and stat/lstat instead #ifdef _WIN32 #define FSTAT _fstat64 #define STAT _stat64 #else #define _FILE_OFFSET_BITS 64 #define FSTAT fstat #define STAT stat #endif #ifdef _MSC_VER #include #define FILENO(f) _fileno(f) #define READ(fd, buf, n) _read(fd, buf, uint(n)) #define WRITE(fd, buf, n) _write(fd, buf, uint(n)) #else #include #define FILENO(f) fileno(f) #define READ(fd, buf, n) read(fd, buf, n) #define WRITE(fd, buf, n) write(fd, buf, n) #endif using namespace std; using namespace kanzi; struct cContext { kanzi::CompressedOutputStream* pCos; size_t blockSize; void* fos; }; namespace kanzi { // Utility classes to map C FILEs to C++ streams class ofstreambuf FINAL : public streambuf { public: ofstreambuf(int fd) : _fd(fd), _buffer(65536) { // Initialize put pointers to the beginning of the buffer setp(&_buffer[0], &_buffer[0] + _buffer.size()); } virtual ~ofstreambuf() { // Call the non-virtual implementation directly instead of the virtual sync() flush(); } protected: // Called when the buffer is full virtual int_type overflow(int_type c) { if (flush() == EOF) return EOF; if (c != EOF) { *pptr() = char(c); pbump(1); } return c; } // Called for explicit sync/flush virtual int sync() { return (flush() == EOF) ? -1 : 0; } // Optimized block write virtual streamsize xsputn(const char* s, streamsize n) { streamsize remaining = n; const char* src = s; while (remaining > 0) { streamsize avail = epptr() - pptr(); if (avail >= remaining) { // Fits in current buffer memcpy(pptr(), src, remaining); pbump(int(remaining)); return n; } if (avail > 0) { // Fill the rest of the buffer memcpy(pptr(), src, avail); pbump(int(avail)); src += avail; remaining -= avail; } // Flush full buffer if (flush() == EOF) return n - remaining; // If the remaining chunk is large, write directly to FD to avoid double copy if (remaining >= streamsize(_buffer.size())) { streamsize toWrite = remaining; while (toWrite > 0) { const ptrdiff_t written = ptrdiff_t(WRITE(_fd, src, toWrite)); if (written <= 0) return n - remaining; // Error src += written; toWrite -= streamsize(written); } remaining = 0; } } return n; } private: int _fd; std::vector _buffer; int flush() { ptrdiff_t n = pptr() - pbase(); if (n > 0) { char* dst = pbase(); ptrdiff_t remaining = n; while (remaining > 0) { const ptrdiff_t written = ptrdiff_t(WRITE(_fd, dst, remaining)); if (written <= 0) return EOF; dst += written; remaining -= written; } pbump(-int(n)); // Reset pbump by subtracting the amount written } return 0; } }; class FileOutputStream FINAL : public ostream { private: ofstreambuf _buf; public: FileOutputStream(int fd) : ostream(nullptr), _buf(fd) { rdbuf(&_buf); } }; } // Create internal cContext and CompressedOutputStream KANZI_API int CDECL initCompressor(struct cData* pData, FILE* dst, struct cContext** pCtx) KANZI_NOEXCEPT { if ((pData == nullptr) || (pCtx == nullptr) || (dst == nullptr)) return Error::ERR_INVALID_PARAM; FileOutputStream* fos = nullptr; cContext* cctx = nullptr; try { // Process params const int fd = FILENO(dst); if (fd == -1) return Error::ERR_CREATE_COMPRESSOR; if ((memchr(pData->transform, 0, sizeof(pData->transform)) == nullptr) || (memchr(pData->entropy, 0, sizeof(pData->entropy)) == nullptr)) { return Error::ERR_INVALID_PARAM; } string transform = TransformFactory::getName(TransformFactory::getType(pData->transform)); string entropy = EntropyEncoderFactory::getName(EntropyEncoderFactory::getType(pData->entropy)); if ((transform.length() >= sizeof(pData->transform)) || (entropy.length() >= sizeof(pData->entropy))) { return Error::ERR_INVALID_PARAM; } memset(pData->transform, 0, sizeof(pData->transform)); strncpy(pData->transform, transform.c_str(), sizeof(pData->transform) - 1); memset(pData->entropy, 0, sizeof(pData->entropy)); strncpy(pData->entropy, entropy.c_str(), sizeof(pData->entropy) - 1); pData->blockSize = (pData->blockSize + 15) & -16; *pCtx = nullptr; size_t fileSize = 0; struct STAT sbuf; if (FSTAT(fd, &sbuf) == 0) { fileSize = size_t(sbuf.st_size); } // Create compression stream and update context fos = new FileOutputStream(fd); cctx = new cContext(); cctx->pCos = new CompressedOutputStream(*fos, pData->jobs, pData->entropy, pData->transform, int(pData->blockSize), pData->checksum, uint64(fileSize), #ifdef CONCURRENCY_ENABLED nullptr, #endif pData->headerless != 0); cctx->blockSize = pData->blockSize; cctx->fos = fos; *pCtx = cctx; } catch (const exception&) { if (fos != nullptr) delete fos; if (cctx != nullptr) delete cctx; return Error::ERR_CREATE_COMPRESSOR; } return 0; } KANZI_API int CDECL compress(struct cContext* pCtx, const unsigned char* src, size_t inSize, size_t* outSize) KANZI_NOEXCEPT { if ((pCtx == nullptr) || (outSize == nullptr)) { return Error::ERR_INVALID_PARAM; } if ((src == nullptr) && (inSize != 0)) { return Error::ERR_INVALID_PARAM; } if (inSize > size_t(pCtx->blockSize)) { return Error::ERR_INVALID_PARAM; } *outSize = 0; int res = 0; CompressedOutputStream* pCos = pCtx->pCos; if (pCos == nullptr) { return Error::ERR_INVALID_PARAM; } try { const uint64 w = pCos->getWritten(); pCos->write((const char*)src, streamsize(inSize)); res = pCos->good() ? 0 : Error::ERR_WRITE_FILE; *outSize = int(pCos->getWritten() - w); } catch (const IOException& ioe) { return ioe.error(); } catch (const exception&) { return Error::ERR_UNKNOWN; } return res; } // Cleanup allocated internal data structures KANZI_API int CDECL disposeCompressor(struct cContext** ppCtx, size_t* outSize) KANZI_NOEXCEPT { if ((ppCtx == nullptr) || (*ppCtx == nullptr) || (outSize == nullptr)) return Error::ERR_INVALID_PARAM; *outSize = 0; cContext* pCtx = *ppCtx; CompressedOutputStream* pCos = pCtx->pCos; try { if (pCos != nullptr) { const uint64 w = pCos->getWritten(); pCos->close(); *outSize = int(pCos->getWritten() - w); delete pCos; pCos = nullptr; pCtx->pCos = nullptr; } if (pCtx->fos != nullptr) delete static_cast(pCtx->fos); pCtx->fos = nullptr; delete pCtx; *ppCtx = nullptr; } catch (const IOException& ioe) { if (pCos != nullptr) { delete pCos; pCos = nullptr; pCtx->pCos = nullptr; } if (pCtx->fos != nullptr) delete static_cast(pCtx->fos); delete pCtx; *ppCtx = nullptr; return ioe.error(); } catch (const exception&) { if (pCos != nullptr) { delete pCos; pCos = nullptr; pCtx->pCos = nullptr; } if (pCtx->fos != nullptr) delete static_cast(pCtx->fos); delete pCtx; *ppCtx = nullptr; return Error::ERR_UNKNOWN; } return 0; } KANZI_API unsigned int CDECL getCompressorVersion(void) KANZI_NOEXCEPT { return (KANZI_COMP_VERSION_MAJOR << 16) | (KANZI_COMP_VERSION_MINOR << 8) | KANZI_COMP_VERSION_PATCH; } kanzi-cpp-2.6.0/src/api/Compressor.hpp000066400000000000000000000076361525576310500176450ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Compressor #define knz_Compressor #ifdef _WIN32 #define CDECL __cdecl #ifdef KANZI_EXPORTS #define KANZI_API __declspec(dllexport) #else #define KANZI_API #endif #else #define CDECL #define KANZI_API #endif #include #ifdef __cplusplus #if __cplusplus >= 201103L // C++ 11 or higher #define KANZI_NOEXCEPT noexcept #else #define KANZI_NOEXCEPT #endif #else #define KANZI_NOEXCEPT #endif #define KANZI_COMP_VERSION_MAJOR 1 #define KANZI_COMP_VERSION_MINOR 0 #define KANZI_COMP_VERSION_PATCH 0 #ifdef __cplusplus extern "C" { #endif /** * Compression context: encapsulates compressor state (opaque: could change in future versions) */ struct cContext; /** * Compression parameters */ struct cData { char transform[64]; /* name of transforms [None|PACK|BWT|BWTS|LZ|LZX|LZP|ROLZ|ROLZX] [RLT|ZRLT|MTFT|RANK|SRT|TEXT|MM|EXE|UTF|DNA] */ char entropy[16]; /* name of entropy codec [None|Huffman|ANS0|ANS1|Range|FPAQ|TPAQ|TPAQX|CM] */ size_t blockSize; /* size of block in bytes */ unsigned int jobs; /* max number of concurrent tasks */ int checksum; /* 0, 32 or 64 to indicate size of block checksum */ int headerless; /* bool to indicate if the bitstream has a header (usually set to 0) */ }; /** * @return the version number of the library. * Useful for checking for compatibility at runtime. */ KANZI_API unsigned int CDECL getCompressorVersion(void) KANZI_NOEXCEPT; /** * Initialize the compressor internal states. * * @param cParam [IN|OUT] - the compression parameters, transform and enropy are validated and rewritten * @param dst [IN] - the destination stream of compressed data * @param ctx [IN|OUT] - pointer to the compression context created by the call * * @return 0 in case of success, else see error code in Error.hpp */ KANZI_API int CDECL initCompressor(struct cData* cParam, FILE* dst, struct cContext** ctx) KANZI_NOEXCEPT; /** * Compress a block of data. The compressor must have been initialized. * * @param ctx [IN] - the compression context created during initialization * @param src [IN] - the source block of data to compress * @param inSize [IN] - the size of the source block to compress. * @param outSize [IN|OUT] - the size of the compressed data Updated to reflect the number bytes written to the destination. * * @return 0 in case of success, else see error code in Error.hpp */ KANZI_API int CDECL compress(struct cContext* ctx, const unsigned char* src, size_t inSize, size_t* outSize) KANZI_NOEXCEPT; /** * Dispose the compressor and cleanup memory resources. * * @param ctx [IN] - the compression context created during initialization * @param outSize [IN|OUT] - the number of bytes written to the destination * (the compressor may flush internal data) * * @return 0 in case of success, else see error code in Error.hpp */ KANZI_API int CDECL disposeCompressor(struct cContext** ctx, size_t* outSize) KANZI_NOEXCEPT; #ifdef __cplusplus } #endif #endif kanzi-cpp-2.6.0/src/api/Decompressor.cpp000066400000000000000000000212211525576310500201330ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include "Decompressor.hpp" #include "../types.hpp" #include "../Error.hpp" #include "../io/IOException.hpp" #include "../io/CompressedInputStream.hpp" #include "../transform/TransformFactory.hpp" #include "../entropy/EntropyDecoderFactory.hpp" #ifdef _MSC_VER #include #define FILENO(f) _fileno(f) #define READ(fd, buf, n) _read(fd, buf, uint(n)) #define WRITE(fd, buf, n) _write(fd, buf, uint(n)) #else #include #define FILENO(f) fileno(f) #define READ(fd, buf, n) read(fd, buf, n) #define WRITE(fd, buf, n) write(fd, buf, n) #endif using namespace std; using namespace kanzi; /** * Decompression context: encapsulates decompressor state (opaque: could change in future versions) */ struct dContext { kanzi::CompressedInputStream* pCis; size_t bufferSize; void* fis; }; namespace kanzi { class ifstreambuf FINAL : public streambuf { public: ifstreambuf(int fd) : _fd(fd) { // gptr() = egptr() initially forces underflow() on first read setg(_buffer + 4, _buffer + 4, _buffer + 4); } private: static const int BUF_SIZE = 1024 + 4; int _fd; char _buffer[BUF_SIZE]; virtual int_type underflow() { if (gptr() < egptr()) return traits_type::to_int_type(*gptr()); // Preserve up to 4 characters for putback int putback = int(gptr() - eback()); if (putback > 4) putback = 4; // Only move putback if > 0 if (putback > 0) { std::memmove(_buffer + (4 - putback), gptr() - putback, putback); } // Read new data const int n = int(READ(_fd, _buffer + 4, BUF_SIZE - 4)); if (n <= 0) return EOF; // Reset get pointers: // eback = start of buffer (including putback area) // gptr = first new kanzi::byte // egptr = end of new data setg(_buffer + (4 - putback), _buffer + 4, _buffer + 4 + n); return traits_type::to_int_type(*gptr()); } }; class FileInputStream FINAL : public istream { private: ifstreambuf _buf; public: FileInputStream(int fd) : istream(nullptr), _buf(fd) { rdbuf(&_buf); } }; } // Create internal dContext and CompressedInputStream KANZI_API int CDECL initDecompressor(struct dData* pData, FILE* src, struct dContext** pCtx) KANZI_NOEXCEPT { if ((pData == nullptr) || (pCtx == nullptr) || (src == nullptr)) return Error::ERR_INVALID_PARAM; // Validate buffer size (sanity check against huge allocations, e.g., > 2GB) if (pData->bufferSize > size_t(2) * 1024 * 1024 * 1024) return Error::ERR_INVALID_PARAM; dContext* dctx = nullptr; FileInputStream* fis = nullptr; try { const int fd = FILENO(src); if (fd == -1) return Error::ERR_CREATE_DECOMPRESSOR; if ((pData->headerless != 0) && ((memchr(pData->transform, 0, sizeof(pData->transform)) == nullptr) || (memchr(pData->entropy, 0, sizeof(pData->entropy)) == nullptr))) { return Error::ERR_INVALID_PARAM; } // Create decompression stream and context *pCtx = nullptr; fis = new FileInputStream(fd); dctx = new dContext(); dctx->pCis = nullptr; dctx->fis = nullptr; if (pData->headerless != 0) { // Headerless mode: process params string transform = TransformFactory::getName(TransformFactory::getType(pData->transform)); string entropy = EntropyDecoderFactory::getName(EntropyDecoderFactory::getType(pData->entropy)); // Validate sizes if ((transform.length() >= sizeof(pData->transform)) || (entropy.length() >= sizeof(pData->entropy))) { delete fis; delete dctx; return Error::ERR_INVALID_PARAM; } memset(pData->transform, 0, sizeof(pData->transform)); strncpy(pData->transform, transform.c_str(), sizeof(pData->transform) - 1); memset(pData->entropy, 0, sizeof(pData->entropy)); strncpy(pData->entropy, entropy.c_str(), sizeof(pData->entropy) - 1); pData->blockSize = (pData->blockSize + 15) & -16; dctx->pCis = new CompressedInputStream(*fis, pData->jobs, pData->entropy, pData->transform, pData->blockSize, pData->checksum, pData->originalSize, #ifdef CONCURRENCY_ENABLED nullptr, #endif true, pData->bsVersion); } else { dctx->pCis = new CompressedInputStream(*fis, pData->jobs); } dctx->bufferSize = pData->bufferSize; dctx->fis = fis; *pCtx = dctx; } catch (const exception&) { if (dctx != nullptr) { if (dctx->pCis != nullptr) { delete dctx->pCis; dctx->pCis = nullptr; } delete dctx; dctx = nullptr; } if (fis != nullptr) { delete fis; fis = nullptr; } return Error::ERR_CREATE_DECOMPRESSOR; } return 0; } KANZI_API int CDECL decompress(struct dContext* pCtx, unsigned char* dst, size_t* inSize, size_t* outSize) KANZI_NOEXCEPT { if ((pCtx == nullptr) || (outSize == nullptr)) { return Error::ERR_INVALID_PARAM; } if (*outSize > pCtx->bufferSize) { return Error::ERR_INVALID_PARAM; } if (*outSize == 0) return 0; if (dst == nullptr) { return Error::ERR_INVALID_PARAM; } if (inSize) *inSize = 0; CompressedInputStream* pCis = pCtx->pCis; if (pCis == nullptr) { *outSize = 0; return Error::ERR_INVALID_PARAM; } try { const uint64 r = pCis->getRead(); pCis->read((char*)dst, std::streamsize(*outSize)); if (!pCis->good() && !pCis->eof()) return Error::ERR_READ_FILE; if (inSize) *inSize = size_t(pCis->getRead() - r); *outSize = size_t(pCis->gcount()); } catch (const IOException& ioe) { *outSize = 0; return ioe.error(); } catch (const exception&) { *outSize = 0; return Error::ERR_UNKNOWN; } return 0; } // Cleanup allocated internal data structures KANZI_API int CDECL disposeDecompressor(struct dContext** ppCtx) KANZI_NOEXCEPT { if ((ppCtx == nullptr) || (*ppCtx == nullptr)) return Error::ERR_INVALID_PARAM; dContext* pCtx = *ppCtx; CompressedInputStream* pCis = static_cast(pCtx->pCis); try { if (pCis != nullptr) { pCis->close(); delete pCis; pCis = nullptr; pCtx->pCis = nullptr; } if (pCtx->fis != nullptr) delete (FileInputStream*)pCtx->fis; pCtx->fis = nullptr; delete pCtx; *ppCtx = nullptr; } catch (const IOException& ioe) { if (pCis != nullptr) { delete pCis; pCis = nullptr; pCtx->pCis = nullptr; } if (pCtx->fis != nullptr) delete (FileInputStream*)pCtx->fis; pCtx->fis = nullptr; delete pCtx; *ppCtx = nullptr; return ioe.error(); } catch (const exception&) { if (pCis != nullptr) { delete pCis; pCtx->pCis = nullptr; } if (pCtx->fis != nullptr) delete (FileInputStream*)pCtx->fis; pCtx->fis = nullptr; delete pCtx; *ppCtx = nullptr; return Error::ERR_UNKNOWN; } return 0; } KANZI_API unsigned int CDECL getDecompressorVersion(void) KANZI_NOEXCEPT { return (KANZI_DECOMP_VERSION_MAJOR << 16) | (KANZI_DECOMP_VERSION_MINOR << 8) | KANZI_DECOMP_VERSION_PATCH; } kanzi-cpp-2.6.0/src/api/Decompressor.hpp000066400000000000000000000101521525576310500201410ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Decompressor #define knz_Decompressor #ifdef _WIN32 #define CDECL __cdecl #ifdef KANZI_EXPORTS #define KANZI_API __declspec(dllexport) #else #define KANZI_API #endif #else #define CDECL #define KANZI_API #endif #include #ifdef __cplusplus #if __cplusplus >= 201103L // C++ 11 or higher #define KANZI_NOEXCEPT noexcept #else #define KANZI_NOEXCEPT #endif #else #define KANZI_NOEXCEPT #endif #define KANZI_DECOMP_VERSION_MAJOR 1 #define KANZI_DECOMP_VERSION_MINOR 0 #define KANZI_DECOMP_VERSION_PATCH 0 #ifdef __cplusplus extern "C" { #endif /** * Decompression context: encapsulates decompressor state (opaque: could change in future versions) */ struct dContext; /** * Decompression parameters */ struct dData { // Required fields size_t bufferSize; /* read buffer size (at least block size) */ unsigned int jobs; /* max number of concurrent tasks */ int headerless; /* bool to indicate if the bitstream has a header (usually set to 0) */ // Optional fields: only required if headerless is true char transform[64]; /* name of transforms [None|PACK|BWT|BWTS|LZ|LZX|LZP|ROLZ|ROLZX] [RLT|ZRLT|MTFT|RANK|SRT|TEXT|MM|EXE|UTF|DNA] */ char entropy[16]; /* name of entropy codec [None|Huffman|ANS0|ANS1|Range|FPAQ|TPAQ|TPAQX|CM] */ unsigned int blockSize; /* size of block in bytes */ size_t originalSize; /* size of original file in bytes */ int checksum; /* 0, 32 or 64 to indicate size of block checksum */ int bsVersion; /* version of the bitstream */ }; /** * @return the version number of the library. * Useful for checking for compatibility at runtime. */ KANZI_API unsigned int CDECL getDecompressorVersion(void) KANZI_NOEXCEPT; /** * Initialize the decompressor internal states. * * @param dParam [IN|OUT] - the decompression parameters. Transform and entropy are * validated and rewritten. * @param src [IN] - the source stream of compressed data * @param ctx [IN|OUT] - a pointer to the decompression context created by the call * * @return 0 in case of success, else see error code in Error.hpp */ KANZI_API int CDECL initDecompressor(struct dData* dParam, FILE* src, struct dContext** ctx) KANZI_NOEXCEPT; /** * Decompress a block of data. The decompressor must have been initialized. * * @param ctx [IN] - the decompression context created during initialization * @param dst [IN] - the destination block of decompressed data * @param inSize [OUT] - the number of bytes read from source. * @param outSize [IN|OUT] - the size of the block to decompress. * Updated to reflect the number of decompressed bytes * * @return 0 in case of success, else see error code in Error.hpp */ KANZI_API int CDECL decompress(struct dContext* ctx, unsigned char* dst, size_t* inSize, size_t* outSize) KANZI_NOEXCEPT; /** * Dispose the decompressor and cleanup memory resources. * * @param ctx [IN] - the compression context created during initialization * * @return 0 in case of success, else see error code in Error.hpp */ KANZI_API int CDECL disposeDecompressor(struct dContext** ctx) KANZI_NOEXCEPT; #ifdef __cplusplus } #endif #endif kanzi-cpp-2.6.0/src/api/kanzi.py000066400000000000000000000075061525576310500164620ustar00rootroot00000000000000import ctypes from kanzi_c_api import _lib, _libc, cContext_p, dContext_p, cData, dData class KanziError(RuntimeError): pass def _check(rc, msg): if rc != 0: raise KanziError(f"{msg} (error code {rc})") # ----------------------------------------------------------------------------- # Compressor # ----------------------------------------------------------------------------- class Compressor: def __init__( self, dst_path, transform=b"LZ", entropy=b"Huffman", block_size=1 << 20, jobs=1, checksum=0, headerless=0, ): self._file = _libc.fopen(dst_path.encode(), b"wb") if not self._file: raise OSError("fopen failed") self._ctx = cContext_p() params = cData() params.transform = transform params.entropy = entropy params.blockSize = block_size params.jobs = jobs params.checksum = checksum params.headerless = headerless rc = _lib.initCompressor( ctypes.byref(params), self._file, ctypes.byref(self._ctx), ) _check(rc, "initCompressor failed") def compress(self, data: bytes) -> int: src = (ctypes.c_ubyte * len(data)).from_buffer_copy(data) out_size = ctypes.c_size_t(0) rc = _lib.compress( self._ctx, src, len(data), ctypes.byref(out_size), ) _check(rc, "compress failed") return out_size.value def close(self) -> int: out_size = ctypes.c_size_t(0) rc = _lib.disposeCompressor( ctypes.byref(self._ctx), ctypes.byref(out_size), ) _check(rc, "disposeCompressor failed") _libc.fclose(self._file) self._file = None return out_size.value def __enter__(self): return self def __exit__(self, exc_type, exc, tb): self.close() # ----------------------------------------------------------------------------- # Decompressor # ----------------------------------------------------------------------------- class Decompressor: def __init__( self, src_path, buffer_size, jobs=1, headerless=0, **headerless_params, ): self._file = _libc.fopen(src_path.encode(), b"rb") if not self._file: raise OSError("fopen failed") self._ctx = dContext_p() params = dData() params.bufferSize = buffer_size params.jobs = jobs params.headerless = headerless if headerless: params.transform = headerless_params["transform"] params.entropy = headerless_params["entropy"] params.blockSize = headerless_params["blockSize"] params.originalSize = headerless_params["originalSize"] params.checksum = headerless_params["checksum"] params.bsVersion = headerless_params["bsVersion"] rc = _lib.initDecompressor( ctypes.byref(params), self._file, ctypes.byref(self._ctx), ) _check(rc, "initDecompressor failed") def decompress_block(self, max_output: int) -> bytes: dst = (ctypes.c_ubyte * max_output)() in_size = ctypes.c_size_t(0) out_size = ctypes.c_size_t(max_output) rc = _lib.decompress( self._ctx, dst, ctypes.byref(in_size), ctypes.byref(out_size), ) _check(rc, "decompress failed") return bytes(dst[: out_size.value]) def close(self): rc = _lib.disposeDecompressor(ctypes.byref(self._ctx)) _check(rc, "disposeDecompressor failed") _libc.fclose(self._file) def __enter__(self): return self def __exit__(self, exc_type, exc, tb): self.close() kanzi-cpp-2.6.0/src/api/kanzi_c_api.py000066400000000000000000000111201525576310500176000ustar00rootroot00000000000000import ctypes import sys import os # ----------------------------------------------------------------------------- # Platform detection # ----------------------------------------------------------------------------- if sys.platform.startswith("win"): KANZI_LIB_NAME = "kanzi.dll" LIBC_NAME = "msvcrt.dll" elif sys.platform == "darwin": KANZI_LIB_NAME = "libkanzi.dylib" LIBC_NAME = "libc.dylib" else: KANZI_LIB_NAME = "libkanzi.so" LIBC_NAME = "libc.so.6" # ----------------------------------------------------------------------------- # Load shared libraries # ----------------------------------------------------------------------------- _lib = ctypes.CDLL(KANZI_LIB_NAME) _libc = ctypes.CDLL(LIBC_NAME) # ----------------------------------------------------------------------------- # libc FILE* # ----------------------------------------------------------------------------- FILE_p = ctypes.c_void_p _libc.fopen.argtypes = [ctypes.c_char_p, ctypes.c_char_p] _libc.fopen.restype = FILE_p _libc.fclose.argtypes = [FILE_p] _libc.fclose.restype = ctypes.c_int # ----------------------------------------------------------------------------- # Opaque contexts # ----------------------------------------------------------------------------- class cContext(ctypes.Structure): pass class dContext(ctypes.Structure): pass cContext_p = ctypes.POINTER(cContext) dContext_p = ctypes.POINTER(dContext) # ----------------------------------------------------------------------------- # Compression parameters # ----------------------------------------------------------------------------- class cData(ctypes.Structure): _fields_ = [ ("transform", ctypes.c_char * 64), ("entropy", ctypes.c_char * 16), ("blockSize", ctypes.c_size_t), ("jobs", ctypes.c_uint), ("checksum", ctypes.c_int), ("headerless", ctypes.c_int), ] class dData(ctypes.Structure): _fields_ = [ # Required ("bufferSize", ctypes.c_size_t), ("jobs", ctypes.c_uint), ("headerless", ctypes.c_int), # Headerless-only ("transform", ctypes.c_char * 64), ("entropy", ctypes.c_char * 16), ("blockSize", ctypes.c_uint), ("originalSize", ctypes.c_size_t), ("checksum", ctypes.c_int), ("bsVersion", ctypes.c_int), ] # ----------------------------------------------------------------------------- # Function prototypes - Compressor # ----------------------------------------------------------------------------- _lib.getCompressorVersion.argtypes = [] _lib.getCompressorVersion.restype = ctypes.c_uint _lib.initCompressor.argtypes = [ ctypes.POINTER(cData), FILE_p, ctypes.POINTER(cContext_p), ] _lib.initCompressor.restype = ctypes.c_int _lib.compress.argtypes = [ cContext_p, ctypes.POINTER(ctypes.c_ubyte), ctypes.c_size_t, ctypes.POINTER(ctypes.c_size_t), ] _lib.compress.restype = ctypes.c_int _lib.disposeCompressor.argtypes = [ ctypes.POINTER(cContext_p), ctypes.POINTER(ctypes.c_size_t), ] _lib.disposeCompressor.restype = ctypes.c_int # ----------------------------------------------------------------------------- # Function prototypes - Decompressor # ----------------------------------------------------------------------------- _lib.getDecompressorVersion.argtypes = [] _lib.getDecompressorVersion.restype = ctypes.c_uint _lib.initDecompressor.argtypes = [ ctypes.POINTER(dData), FILE_p, ctypes.POINTER(dContext_p), ] _lib.initDecompressor.restype = ctypes.c_int _lib.decompress.argtypes = [ dContext_p, ctypes.POINTER(ctypes.c_ubyte), ctypes.POINTER(ctypes.c_size_t), ctypes.POINTER(ctypes.c_size_t), ] _lib.decompress.restype = ctypes.c_int _lib.disposeDecompressor.argtypes = [ ctypes.POINTER(dContext_p), ] _lib.disposeDecompressor.restype = ctypes.c_int # ----------------------------------------------------------------------------- # Optional helpers (recommended for kanzi.py) # ----------------------------------------------------------------------------- def fopen(path: bytes, mode: bytes) -> FILE_p: return _libc.fopen(path, mode) def fclose(fp: FILE_p) -> int: return _libc.fclose(fp) # ----------------------------------------------------------------------------- # Public exports # ----------------------------------------------------------------------------- __all__ = [ # libraries "_lib", "_libc", # FILE* "FILE_p", "fopen", "fclose", # contexts "cContext", "dContext", "cContext_p", "dContext_p", # params "cData", "dData", # functions (via _lib) ] kanzi-cpp-2.6.0/src/app/000077500000000000000000000000001525576310500147735ustar00rootroot00000000000000kanzi-cpp-2.6.0/src/app/BlockCompressor.cpp000066400000000000000000001026011525576310500206060ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include "BlockCompressor.hpp" #include "InfoPrinter.hpp" #include "../SliceArray.hpp" #include "../transform/TransformFactory.hpp" #include "../io/IOException.hpp" #include "../io/IOUtil.hpp" #include "../io/NullOutputStream.hpp" #include "../util/Clock.hpp" #include "../util/Printer.hpp" #ifdef CONCURRENCY_ENABLED #include #endif using namespace kanzi; using namespace std; const int BlockCompressor::DEFAULT_BLOCK_SIZE = 4 * 1024 * 1024; const int BlockCompressor::MIN_BLOCK_SIZE = 1024; const int BlockCompressor::MAX_BLOCK_SIZE = 1024 * 1024 * 1024; BlockCompressor::BlockCompressor(const Context& ctx) : _ctx(ctx) { int level = -1; if (_ctx.has("level") == true) { level = _ctx.getInt("level"); if ((level < 0) || (level > 9)) throw invalid_argument("Invalid compression level"); string tranformAndCodec[2]; getTransformAndCodec(level, tranformAndCodec); _transform = tranformAndCodec[0]; _codec = tranformAndCodec[1]; } else { if ((_ctx.has("transform") == false) && (_ctx.has("entropy") == false)) { // Default to level 3 string tranformAndCodec[2]; getTransformAndCodec(3, tranformAndCodec); _transform = tranformAndCodec[0]; _codec = tranformAndCodec[1]; } else { _codec = _ctx.getString("entropy", "NONE"); string strTransf = _ctx.getString("transform", "NONE"); // Extract transform names. Curate input (EG. NONE+NONE+xxxx => xxxx) _transform = TransformFactory::getName(TransformFactory::getType(strTransf.c_str())); } } _checksum = _ctx.getInt("checksum", 0); if ((_checksum != 0) && (_checksum != 32) && (_checksum != 64)) throw invalid_argument("Invalid block checksum size"); _ctx.putInt("checksum", _checksum); _ctx.putString("entropy", _codec); _ctx.putString("transform", _transform); _overwrite = _ctx.getInt("overwrite", 0) != 0; _ctx.putInt("overwrite", _overwrite ? 1 : 0); _skipBlocks = _ctx.getInt("skipBlocks", 0) != 0; _ctx.putInt("skipBlocks", _skipBlocks ? 1 : 0); _verbosity = _ctx.getInt("verbosity", 1); _ctx.putInt("verbosity", _verbosity); _jobs = _ctx.getInt("jobs", 1); _ctx.putInt("jobs", _jobs); _noDotFiles = _ctx.getInt("noDotFiles", 0) != 0; _ctx.putInt("noDotFiles", _noDotFiles ? 1 : 0); _noLinks = _ctx.getInt("noLinks", 0) != 0; _ctx.putInt("noLinks", _noLinks ? 1 : 0); _autoBlockSize = _ctx.getInt("autoBlock", 0) != 0; _ctx.putInt("autoBlock", _autoBlockSize ? 1 : 0); _reorderFiles = _ctx.getInt("fileReorder", 0) != 0; _ctx.putInt("fileReorder", _reorderFiles ? 1 : 0); bool remove = _ctx.getInt("remove", 0) != 0; _ctx.putInt("remove", remove ? 1 : 0); if (_ctx.has("inputName") == false) throw invalid_argument("Missing input name"); _inputName = _ctx.getString("inputName") == "" ? "STDIN" : _ctx.getString("inputName"); if (Global::isReservedName(_inputName)) throw invalid_argument("'" + _inputName + "' is a reserved name"); if (_ctx.has("outputName") == false) throw invalid_argument("Missing output name"); string str = _ctx.getString("outputName"); _outputName = (str == "") && (_inputName == "STDIN") ? "STDOUT" : str; if (Global::isReservedName(_outputName)) throw invalid_argument("'" + _outputName + "' is a reserved name"); if (_ctx.has("blockSize") == false) { switch (level) { case 6: _blockSize = 2 * DEFAULT_BLOCK_SIZE; break; case 7: _blockSize = 4 * DEFAULT_BLOCK_SIZE; break; case 8: _blockSize = 4 * DEFAULT_BLOCK_SIZE; break; case 9: _blockSize = 8 * DEFAULT_BLOCK_SIZE; break; default: _blockSize = DEFAULT_BLOCK_SIZE; } _ctx.putInt("blockSize", _blockSize); } else { uint64 bl = _ctx.getLong("blockSize", DEFAULT_BLOCK_SIZE); if (bl < MIN_BLOCK_SIZE) { stringstream sserr; sserr << "Minimum block size is " << (MIN_BLOCK_SIZE / 1024) << " KiB ("; sserr << MIN_BLOCK_SIZE << " bytes), got " << bl; sserr << (bl > 1 ? " bytes" : " byte"); throw invalid_argument(sserr.str()); } if (bl > MAX_BLOCK_SIZE) { stringstream sserr; sserr << "Maximum block size is " << (MAX_BLOCK_SIZE / (1024 * 1024 * 1024)) << " GiB ("; sserr << MAX_BLOCK_SIZE << " bytes), got " << bl << " bytes"; throw invalid_argument(sserr.str()); } bl = (bl + 15) & ~uint64(15); _blockSize = int(min(bl, uint64(MAX_BLOCK_SIZE))); } } BlockCompressor::~BlockCompressor() { dispose(); _listeners.clear(); } int BlockCompressor::compress(uint64& outputSize) { vector files; Clock stopClock; int nbFiles = 1; Printer log(cout); stringstream ss; string upperInputName = _inputName; transform(upperInputName.begin(), upperInputName.end(), upperInputName.begin(), safeToUpper); bool isStdIn = upperInputName == "STDIN"; if (isStdIn == false) { vector errors; bool isRecursive = (_inputName.length() < 2) || (_inputName[_inputName.length() - 2] != PATH_SEPARATOR) || (_inputName[_inputName.length() - 1] != '.'); FileListConfig cfg = { isRecursive, _noLinks, false, _noDotFiles }; createFileList(_inputName, files, cfg, errors); if (errors.size() > 0) { for (size_t i = 0; i < errors.size(); i++) cerr << errors[i] << endl; return Error::ERR_OPEN_FILE; } if (files.size() == 0) { cerr << "Cannot find any file to compress" << endl; return Error::ERR_OPEN_FILE; } nbFiles = int(files.size()); string strFiles = nbFiles > 1 ? " files" : " file"; ss << nbFiles << strFiles << " to compress\n"; log.println(ss.str(), _verbosity > 0); ss.str(string()); } string upperOutputName = _outputName; transform(upperOutputName.begin(), upperOutputName.end(), upperOutputName.begin(), safeToUpper); bool isStdOut = upperOutputName == "STDOUT"; // Limit verbosity level when output is stdout // Logic is duplicated here to avoid dependency to Kanzi.cpp if (isStdOut == true) _verbosity = 0; // Limit verbosity level when files are processed concurrently if ((_verbosity > 1) && (_jobs > 1) && (nbFiles > 1)) { log.println("Warning: limiting verbosity to 1 due to concurrent processing of input files.\n", true); _verbosity = 1; } if (_verbosity > 2) { if (_autoBlockSize == true) ss << "Block size: 'auto'" << endl; else ss << "Block size: " << _blockSize << " bytes" << endl; ss << "Verbosity: " << _verbosity << endl; ss << "Overwrite: " << (_overwrite ? "true" : "false") << endl; string ckSize = "NONE"; if (_checksum == 32) ckSize = "32 bits"; else if (_checksum == 64) ckSize = "64 bits"; ss << "Block checksum: " << ckSize << endl; string etransform = _transform; transform(etransform.begin(), etransform.end(), etransform.begin(), safeToUpper); ss << "Using " << (etransform == "NONE" ? "no" : _transform) << " transform (stage 1)" << endl; string ecodec = _codec; transform(ecodec.begin(), ecodec.end(), ecodec.begin(), safeToUpper); ss << "Using " << (ecodec == "NONE" ? "no" : _codec) << " entropy codec (stage 2)" << endl; ss << "Using " << _jobs << " job" << (_jobs > 1 ? "s" : "") << endl; log.print(ss.str(), true); ss.str(string()); } InfoPrinter listener(_verbosity, InfoPrinter::COMPRESSION, cout); if (_verbosity > 2) addListener(listener); int res = 0; uint64 read = 0; uint64 written = 0; bool inputIsDir = false; string formattedOutName = _outputName; string formattedInName = _inputName; bool specialOutput = (isStdOut == true) || (upperOutputName == "NONE"); // Need to strip path separator at the end to make 'stat()' happy if ((formattedOutName.size() > 1) && (formattedOutName[formattedOutName.size() - 1] == PATH_SEPARATOR)) { formattedOutName.resize(formattedOutName.size() - 1); } if (isStdIn == false) { struct STAT buffer; if ((formattedInName.size() > 1) && (formattedInName[formattedInName.size() - 1] == PATH_SEPARATOR)) { formattedInName.resize(formattedInName.size() - 1); } if (STAT(formattedInName.c_str(), &buffer) != 0) { cerr << "Cannot access input file '" << formattedInName << "'" << endl; return Error::ERR_OPEN_FILE; } if ((buffer.st_mode & S_IFDIR) != 0) { inputIsDir = true; if ((formattedInName.size() != 0) && (formattedInName[formattedInName.size() - 1] == '.')) { formattedInName.resize(formattedInName.size() - 1); } if ((formattedInName.size() != 0) && (formattedInName[formattedInName.size() - 1] != PATH_SEPARATOR)) { formattedInName += PATH_SEPARATOR; } if ((formattedOutName.size() != 0) && (specialOutput == false)) { if (STAT(formattedOutName.c_str(), &buffer) != 0) { cerr << "Output must be an existing directory (or 'NONE')" << endl; return Error::ERR_OPEN_FILE; } if ((buffer.st_mode & S_IFDIR) == 0) { cerr << "Output must be a directory (or 'NONE')" << endl; return Error::ERR_CREATE_FILE; } formattedOutName += PATH_SEPARATOR; } } else { if ((formattedOutName.size() != 0) && (specialOutput == false)) { if ((STAT(formattedOutName.c_str(), &buffer) == 0) && ((buffer.st_mode & S_IFDIR) != 0)) { cerr << "Output must be a file (or 'NONE')" << endl; return Error::ERR_CREATE_FILE; } } } } #ifdef CONCURRENCY_ENABLED ThreadPool pool(_jobs + 1); // +1 to avoid deadlock due to thread exhaustion #endif _ctx.putInt("verbosity", _verbosity); // Run the task(s) if (nbFiles == 1) { string oName = formattedOutName; string iName = "STDIN"; if (isStdIn == true) { if (oName.length() == 0) { oName = "STDOUT"; } } else { iName = files[0].fullPath(); _ctx.putLong("fileSize", files[0]._size); // Set the block size to optimize compression ratio when possible if ((_autoBlockSize == true) && (_jobs > 0)) { const int64 bl = files[0]._size / _jobs; _blockSize = int(max(min((bl + 63) & ~63, int64(MAX_BLOCK_SIZE)), int64(MIN_BLOCK_SIZE))); _ctx.putInt("blockSize", _blockSize); } if (oName.length() == 0) { oName = iName + ".knz"; } else if ((inputIsDir == true) && (specialOutput == false)) { oName = formattedOutName + iName.substr(formattedInName.size()) + ".knz"; } } _ctx.putString("inputName", iName); _ctx.putString("outputName", oName); FileCompressTask task(_ctx, _listeners); FileCompressResult fcr = task.run(); res = fcr._code; read = fcr._read; written = fcr._written; if (res != 0) { cerr << fcr._errMsg << endl; } } else { vector*> tasks; tasks.reserve(nbFiles); #ifdef CONCURRENCY_ENABLED vector jobsPerTask(nbFiles); vector*> workers; Global::computeJobsPerTask(jobsPerTask.data(), _jobs, nbFiles); #endif try { if (_reorderFiles == true) sortFilesByPathAndSize(files, true); // Create one task per file for (int i = 0; i < nbFiles; i++) { string oName = formattedOutName; string iName = files[i].fullPath(); if (oName.length() == 0) { oName = iName + ".knz"; } else if ((inputIsDir == true) && (specialOutput == false)) { oName = formattedOutName + iName.substr(formattedInName.size()) + ".knz"; } int blockSize = _blockSize; // Set the block size to optimize compression ratio when possible if ((_autoBlockSize == true) && (_jobs > 0)) { const int64 bl = files[i]._size / _jobs; blockSize = int(max(min((bl + 63) & ~63, int64(MAX_BLOCK_SIZE)), int64(MIN_BLOCK_SIZE))); } #ifdef CONCURRENCY_ENABLED Context taskCtx(_ctx, &pool); taskCtx.putInt("jobs", jobsPerTask[i]); #else Context taskCtx(_ctx); taskCtx.putInt("jobs", 1); #endif taskCtx.putLong("fileSize", files[i]._size); taskCtx.putString("inputName", iName); taskCtx.putString("outputName", oName); taskCtx.putInt("blockSize", blockSize); tasks.push_back(new FileCompressTask(taskCtx, _listeners)); } bool doConcurrent = _jobs > 1; #ifdef CONCURRENCY_ENABLED if (doConcurrent) { BoundedConcurrentQueue queue(nbFiles, &tasks[0]); // !tasks.empty() vector > results; workers.reserve(_jobs); results.reserve(_jobs); try { // Create one worker per job and run it. A worker calls several tasks sequentially. for (int i = 0; i < _jobs; i++) { workers.push_back(new FileCompressWorker(&queue)); if (_ctx.getPool() == nullptr) results.push_back(std::async(launch::async, &FileCompressWorker::run, workers[i])); else results.push_back(_ctx.getPool()->schedule(&FileCompressWorker::run, workers[i])); } // Wait for results for (int i = 0; i < _jobs; i++) { FileCompressResult fcr = results[i].get(); read += fcr._read; written += fcr._written; if (fcr._code != 0) { if (res == 0) res = fcr._code; cerr << fcr._errMsg << endl; // Exit early by telling the workers that the queue is empty queue.clear(); } } } catch (...) { queue.clear(); for (uint i = 0; i < results.size(); i++) { try { if (results[i].valid()) results[i].wait(); } catch (const exception&) { } } throw; } } #endif if (!doConcurrent) { for (uint i = 0; i < tasks.size(); i++) { FileCompressResult fcr = tasks[i]->run(); res = fcr._code; read += fcr._read; written += fcr._written; if (res != 0) { cerr << fcr._errMsg << endl; break; } } } } catch (...) { #ifdef CONCURRENCY_ENABLED for (uint i = 0; i < workers.size(); i++) delete workers[i]; #endif for (uint i = 0; i < tasks.size(); i++) delete tasks[i]; throw; } #ifdef CONCURRENCY_ENABLED for (uint i = 0; i < workers.size(); i++) delete workers[i]; #endif for (uint i = 0; i < tasks.size(); i++) delete tasks[i]; } stopClock.stop(); if (nbFiles > 1) { if (_verbosity > 0) { double delta = stopClock.elapsed(); log.println("", true); ss.str(string()); ss << "Total compression time: "; if (delta >= 1e5) { ss.precision(1); ss.setf(ios::fixed); ss << (delta / 1000) << " s" << endl; } else { ss << int(delta) << " ms" << endl; } ss << "Total output size: " << written << (written > 1 ? " bytes" : " byte") << endl; if (read > 0) { ss.precision(6); ss << "Compression ratio: " << (float(written) / float(read)) << endl; } log.print(ss.str(), true); ss.str(string()); } } if (_verbosity > 2) removeListener(listener); outputSize += written; return res; } bool BlockCompressor::addListener(Listener& bl) { _listeners.push_back(&bl); return true; } bool BlockCompressor::removeListener(Listener& bl) { std::vector*>::iterator it = find(_listeners.begin(), _listeners.end(), &bl); if (it == _listeners.end()) return false; _listeners.erase(it); return true; } void BlockCompressor::notifyListeners(vector*>& listeners, const Event& evt) { for (size_t i = 0; i < listeners.size(); i++) listeners[i]->processEvent(evt); } void BlockCompressor::getTransformAndCodec(int level, string tranformAndCodec[2]) { switch (level) { case 0: tranformAndCodec[0] = "NONE"; tranformAndCodec[1] = "NONE"; break; case 1: tranformAndCodec[0] = "LZX"; tranformAndCodec[1] = "NONE"; break; case 2: tranformAndCodec[0] = "DNA+LZ"; tranformAndCodec[1] = "HUFFMAN"; break; case 3: tranformAndCodec[0] = "TEXT+UTF+PACK+MM+LZX"; tranformAndCodec[1] = "HUFFMAN"; break; case 4: tranformAndCodec[0] = "TEXT+UTF+EXE+PACK+MM+ROLZ"; tranformAndCodec[1] = "NONE"; break; case 5: tranformAndCodec[0] = "TEXT+UTF+BWT+RANK+ZRLT"; tranformAndCodec[1] = "ANS0"; break; case 6: tranformAndCodec[0] = "TEXT+UTF+BWT+SRT+ZRLT"; tranformAndCodec[1] = "FPAQ"; break; case 7: tranformAndCodec[0] = "LZP+TEXT+UTF+BWT+LZP"; tranformAndCodec[1] = "CM"; break; case 8: tranformAndCodec[0] = "EXE+RLT+TEXT+UTF+DNA"; tranformAndCodec[1] = "TPAQ"; break; case 9: tranformAndCodec[0] = "EXE+RLT+TEXT+UTF+DNA"; tranformAndCodec[1] = "TPAQX"; break; default: tranformAndCodec[0] = "Unknown"; tranformAndCodec[1] = "Unknown"; } } template FileCompressTask::FileCompressTask(const Context& ctx, vector*>& listeners) : _ctx(ctx) , _listeners(listeners) { _is = nullptr; _cos = nullptr; } template string FileCompressTask::describeStreamState(const std::istream& is) { const ios_base::iostate state = is.rdstate(); if (state == ios_base::goodbit) return "goodbit"; stringstream ss; bool first = true; if ((state & ios_base::eofbit) != 0) { ss << "eofbit"; first = false; } if ((state & ios_base::failbit) != 0) { if (first == false) ss << '|'; ss << "failbit"; first = false; } if ((state & ios_base::badbit) != 0) { if (first == false) ss << '|'; ss << "badbit"; } return ss.str(); } template T FileCompressTask::run() { Printer log(cout); int verbosity = _ctx.getInt("verbosity"); string inputName = _ctx.getString("inputName"); string outputName = _ctx.getString("outputName"); stringstream ss; if (verbosity > 2) { ss << "Input file name: '" << inputName << "'" << endl; ss << "Output file name: '" << outputName << "'" << endl; log.print(ss.str(), true); ss.str(string()); } bool overwrite = _ctx.getInt("overwrite") != 0; OutputStream* os = nullptr; #define CLEANUP_COMP_OS if ((os != nullptr) && (os != &cout)) {\ delete os; \ os = nullptr; \ } try { string str = outputName; transform(str.begin(), str.end(), str.begin(), safeToUpper); if (str == "NONE") { os = new NullOutputStream(); } else if (str == "STDOUT") { os = &cout; } else { if (samePaths(inputName, outputName)) { stringstream sserr; sserr << "The input and output files must be different" << endl; return T(Error::ERR_CREATE_FILE, 0, 0, sserr.str()); } struct STAT buffer; string path = outputName; replace(path.begin(), path.end(), '\\', '/'); if (STAT(outputName.c_str(), &buffer) == 0) { if ((buffer.st_mode & S_IFDIR) != 0) { return T(Error::ERR_OUTPUT_IS_DIR, 0, 0, "The output file is a directory"); } if (overwrite == false) { stringstream sserr; sserr << "File '" << outputName << "' exists and the 'force' command " << "line option has not been provided"; return T(Error::ERR_OVERWRITE_FILE, 0, 0, sserr.str()); } // Delete output file to ensure consistent performance remove(outputName.c_str()); } os = new ofstream(outputName.c_str(), ofstream::out | ofstream::binary); if (!*os) { string errMsg; if (overwrite == true) { // Attempt to create the full folder hierarchy to file string parentDir = outputName; size_t idx = outputName.find_last_of(PATH_SEPARATOR); if (idx != string::npos) parentDir.resize(idx); int rmkd = mkdirAll(parentDir); if ((rmkd == 0) || (rmkd == EEXIST)) { delete os; os = new ofstream(outputName.c_str(), ofstream::out | ofstream::binary); } else { errMsg = strerror(rmkd); } } if (!*os) { delete os; stringstream sserr; sserr << "Cannot open output file '" << outputName << "' for writing"; if (errMsg != "") sserr << ": " << errMsg; return T(Error::ERR_CREATE_FILE, 0, 0, sserr.str()); } } } try { _cos = new CompressedOutputStream(*os, _ctx); for (uint i = 0; i < _listeners.size(); i++) _cos->addListener(*_listeners[i]); } catch (const invalid_argument& e) { CLEANUP_COMP_OS stringstream sserr; sserr << "Cannot create compressed stream: " << e.what(); return T(Error::ERR_CREATE_COMPRESSOR, 0, 0, sserr.str()); } } catch (const exception& e) { CLEANUP_COMP_OS stringstream sserr; sserr << "Cannot open output file '" << outputName << "' for writing: " << e.what(); return T(Error::ERR_CREATE_FILE, 0, 0, sserr.str()); } #define CLEANUP_COMP_IS if ((_is != nullptr) && (_is != &cin)) {\ delete _is; \ _is = nullptr; \ } try { string str = inputName; transform(str.begin(), str.end(), str.begin(), safeToUpper); if (str == "STDIN") { _is = &cin; } else { ifstream* ifs = new ifstream(inputName.c_str(), ifstream::in | ifstream::binary); if (!*ifs) { delete ifs; delete _cos; _cos = nullptr; stringstream sserr; sserr << "Cannot open input file '" << inputName << "'"; return T(Error::ERR_OPEN_FILE, 0, 0, sserr.str()); } _is = ifs; } } catch (const exception& e) { delete _cos; _cos = nullptr; CLEANUP_COMP_IS CLEANUP_COMP_OS stringstream sserr; sserr << "Cannot open input file '" << inputName << "': " << e.what(); return T(Error::ERR_OPEN_FILE, 0, 0, sserr.str()); } // Compress ss << "\nCompressing " << inputName << " ..."; log.println(ss.str(), verbosity > 1); log.println("\n", verbosity > 3); int64 read = 0; kanzi::byte* buf = new kanzi::byte[DEFAULT_BUFFER_SIZE]; SliceArray sa(buf, DEFAULT_BUFFER_SIZE, 0); WallTimer timer; if (_listeners.size() > 0) { int64 inputSize = _ctx.getLong("fileSize", -1); Event evt(Event::COMPRESSION_START, 0, inputSize, timer.getCurrentTime()); BlockCompressor::notifyListeners(_listeners, evt); } Clock stopClock; try { while (true) { int len; try { _is->read(reinterpret_cast(&sa._array[0]), sa._length); len = int(_is->gcount()); if ((_is->bad() == true) || ((_is->fail() == true) && (_is->eof() == false))) { const string state = FileCompressTask::describeStreamState(*_is); CLEANUP_COMP_IS const uint64 w = _cos->getWritten(); delete[] buf; delete _cos; _cos = nullptr; CLEANUP_COMP_OS stringstream sserr; sserr << "Failed to read block from file '" << inputName << "'"; sserr << " (stream state: " << state << ")"; return T(Error::ERR_READ_FILE, read, w, sserr.str().c_str()); } } catch (const exception& e) { CLEANUP_COMP_IS const uint64 w = _cos->getWritten(); delete[] buf; delete _cos; _cos = nullptr; CLEANUP_COMP_OS stringstream sserr; sserr << "Failed to read block from file '" << inputName << "': "; sserr << e.what() << endl; return T(Error::ERR_READ_FILE, read, w, sserr.str().c_str()); } if (len <= 0) break; // Just write block to the compressed output stream ! read += len; _cos->write(reinterpret_cast(&sa._array[0]), len); } } catch (const IOException& ioe) { const uint64 w = _cos->getWritten(); delete _cos; _cos = nullptr; CLEANUP_COMP_IS CLEANUP_COMP_OS delete[] buf; return T(ioe.error(), read, w, ioe.what()); } catch (const exception& e) { const uint64 w = _cos->getWritten(); delete _cos; _cos = nullptr; CLEANUP_COMP_IS CLEANUP_COMP_OS delete[] buf; stringstream sserr; sserr << "An unexpected condition happened. Exiting ..." << endl << e.what(); return T(Error::ERR_UNKNOWN, read, w, sserr.str()); } // Close compressed stream to ensure all data are flushed try { if (_cos != nullptr) _cos->close(); } catch (const IOException& ioe) { const uint64 w = (_cos != nullptr) ? _cos->getWritten() : 0; if (_cos != nullptr) { delete _cos; _cos = nullptr; } CLEANUP_COMP_OS CLEANUP_COMP_IS delete[] buf; return T(ioe.error(), read, w, ioe.what()); } catch (const exception& e) { const uint64 w = (_cos != nullptr) ? _cos->getWritten() : 0; if (_cos != nullptr) { delete _cos; _cos = nullptr; } CLEANUP_COMP_OS CLEANUP_COMP_IS delete[] buf; stringstream sserr; sserr << "Compression failure: " << e.what(); return T(Error::ERR_UNKNOWN, read, w, sserr.str()); } uint64 encoded = _cos->getWritten(); // Clean up resources at the end of the method as the task may be // recycled in a threadpool and the destructor not called. delete _cos; _cos = nullptr; // os destructor will call close if ofstream CLEANUP_COMP_OS CLEANUP_COMP_IS stopClock.stop(); double delta = stopClock.elapsed(); if (verbosity >= 1) { log.println("", (verbosity > 1) && (read > 0)); ss.str(string()); if (verbosity > 1) { if (delta >= 1e5) { ss.precision(1); ss.setf(ios::fixed); ss << "Compression time: " << (delta / 1000) << " s" << endl; } else { ss << "Compression time: " << int(delta) << " ms" << endl; } ss << "Input size: " << read << endl; ss << "Output size: " << encoded << endl; if (read > 0) { ss.precision(6); ss << "Compression ratio: " << (double(encoded) / double(read)) << endl; } log.print(ss.str(), true); ss.str(string()); } else { ss << "Compressed " << inputName << ": " << read << " => " << encoded; ss.precision(2); ss.setf(ios::fixed); if (read > 0) { const double r = double(encoded) / double(read); ss << " (" << (100 * r) << "%)"; } if (delta >= 1e5) { ss.precision(1); ss << " in " << (delta / 1000) << " s"; } else { ss << " in " << int(delta) << " ms"; } log.println(ss.str(), true); ss.str(string()); } if ((verbosity > 1) && (delta > 0) && (read > 0)) { double b2KiB = double(1000) / double(1024); ss << "Throughput (KiB/s): " << uint(double(read) * b2KiB / delta); log.println(ss.str(), true); ss.str(string()); } log.println("", verbosity > 1); } if (_listeners.size() > 0) { Event evt(Event::COMPRESSION_END, 0, int64(encoded), timer.getCurrentTime()); BlockCompressor::notifyListeners(_listeners, evt); } if (_ctx.getInt("remove", 0) != 0) { // Delete input file if (inputName == "STDIN") { log.println("Warning: ignoring remove option with STDIN", verbosity > 0); } else if (remove(inputName.c_str()) != 0) { log.println("Warning: input file could not be deleted", verbosity > 0); } } #undef CLEANUP_COMP_IS #undef CLEANUP_COMP_OS delete[] buf; return T(0, read, encoded, ""); } template FileCompressTask::~FileCompressTask() { dispose(); if (_cos != nullptr) { delete _cos; _cos = nullptr; } try { if ((_is != nullptr) && (_is != &cin)) { delete _is; } _is = nullptr; } catch (const exception&) { // Ignore: best effort } } // Close and flush streams. Do not deallocate resources. Idempotent. template void FileCompressTask::dispose() { try { if (_cos != nullptr) { _cos->close(); } } catch (const exception& e) { cerr << "Compression failure: " << e.what() << endl; } // _is destructor will call close if ifstream } #ifdef CONCURRENCY_ENABLED template R FileCompressWorker::run() { int res = 0; uint64 read = 0; uint64 written = 0; string errMsg; while (res == 0) { T* task = _queue->get(); if (task == nullptr) break; R result = (*task)->run(); res = result._code; read += result._read; written += result._written; if (res != 0) { errMsg += result._errMsg; } } return R(res, read, written, errMsg); } #endif kanzi-cpp-2.6.0/src/app/BlockCompressor.hpp000066400000000000000000000102421525576310500206120ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_BlockCompressor #define knz_BlockCompressor #include #include #include "../InputStream.hpp" #include "../io/CompressedOutputStream.hpp" namespace kanzi { class FileCompressResult { public: int _code; uint64 _read; uint64 _written; std::string _errMsg; FileCompressResult() : _code(0) , _read(0) , _written(0) , _errMsg() { } FileCompressResult(int code, uint64 read, uint64 written, const std::string& errMsg) : _code(code) , _read(read) , _written(written) , _errMsg(errMsg) { } #if __cplusplus < 201103L FileCompressResult(const FileCompressResult& fcr) : _code(fcr._code) , _read(fcr._read) , _written(fcr._written) , _errMsg(fcr._errMsg) { } FileCompressResult& operator=(const FileCompressResult& fcr) { _errMsg = fcr._errMsg; _code = fcr._code; _read = fcr._read; _written = fcr._written; return *this; } ~FileCompressResult() {} #else FileCompressResult(const FileCompressResult& fdr) = delete; FileCompressResult& operator=(const FileCompressResult& fdr) = delete; FileCompressResult(FileCompressResult&& fdr) = default; FileCompressResult& operator=(FileCompressResult&& fdr) = default; ~FileCompressResult() = default; #endif }; #ifdef CONCURRENCY_ENABLED template class FileCompressWorker FINAL : public Task { public: FileCompressWorker(BoundedConcurrentQueue* queue) : _queue(queue) { } ~FileCompressWorker() {} R run(); private: BoundedConcurrentQueue* _queue; }; #endif template class FileCompressTask FINAL : public Task { public: static const int DEFAULT_BUFFER_SIZE = 65536; FileCompressTask(const Context& ctx, std::vector*>& listeners); ~FileCompressTask(); T run(); void dispose(); private: static std::string describeStreamState(const std::istream& is); Context _ctx; InputStream* _is; CompressedOutputStream* _cos; std::vector*> _listeners; }; typedef FileCompressTask FCTask; class BlockCompressor { friend class FileCompressTask; public: BlockCompressor(const Context& ctx); ~BlockCompressor(); int compress(uint64& written); // Register a copy of the listener // Not thread safe bool addListener(Listener& bl); // Not thread safe bool removeListener(Listener& bl); void dispose() const {}; private: static const int DEFAULT_BLOCK_SIZE; static const int MIN_BLOCK_SIZE; static const int MAX_BLOCK_SIZE; int _verbosity; int _checksum; bool _overwrite; bool _skipBlocks; std::string _inputName; std::string _outputName; std::string _codec; std::string _transform; int _blockSize; bool _autoBlockSize; // derive block size from input size and jobs int _jobs; std::vector*> _listeners; bool _reorderFiles; bool _noDotFiles; bool _noLinks; Context _ctx; static void notifyListeners(std::vector*>& listeners, const Event& evt); static void getTransformAndCodec(int level, std::string tranformAndCodec[2]); }; } #endif kanzi-cpp-2.6.0/src/app/BlockDecompressor.cpp000066400000000000000000000735471525576310500211370ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include "BlockDecompressor.hpp" #include "InfoPrinter.hpp" #include "../Global.hpp" #include "../SliceArray.hpp" #include "../io/IOException.hpp" #include "../io/IOUtil.hpp" #include "../io/NullOutputStream.hpp" #include "../util/Clock.hpp" #include "../util/Printer.hpp" #ifdef CONCURRENCY_ENABLED #include #endif using namespace kanzi; using namespace std; BlockDecompressor::BlockDecompressor(const Context& ctx) : _ctx(ctx) { _blockSize = 0; _overwrite = _ctx.getInt("overwrite", 0) != 0; _ctx.putInt("overwrite", _overwrite ? 1 : 0); _verbosity = _ctx.getInt("verbosity", 1); _ctx.putInt("verbosity", _verbosity); _jobs = _ctx.getInt("jobs", 1); _ctx.putInt("jobs", _jobs); bool remove = _ctx.getInt("remove", 0) != 0; _ctx.putInt("remove", remove ? 1 : 0); _noDotFiles = _ctx.getInt("noDotFiles", 0) != 0; _ctx.putInt("noDotFiles", _noDotFiles ? 1 : 0); _noLinks = _ctx.getInt("noLinks", 0) != 0; _ctx.putInt("noLinks", _noLinks ? 1 : 0); if (_ctx.has("inputName") == false) throw invalid_argument("Missing input name"); _inputName = _ctx.getString("inputName") == "" ? "STDIN" : _ctx.getString("inputName"); if (Global::isReservedName(_inputName)) throw invalid_argument("'" + _inputName + "' is a reserved name"); if (_ctx.has("outputName") == false) throw invalid_argument("Missing output name"); string str = _ctx.getString("outputName"); _outputName = (str == "") && (_inputName == "STDIN") ? "STDOUT" : str; if (Global::isReservedName(_outputName)) throw invalid_argument("'" + _outputName + "' is a reserved name"); } BlockDecompressor::~BlockDecompressor() { dispose(); _listeners.clear(); } int BlockDecompressor::decompress(uint64& inputSize) { vector files; uint64 read = 0; Clock stopClock; int nbFiles = 1; Printer log(cout); stringstream ss; string upperInputName = _inputName; transform(upperInputName.begin(), upperInputName.end(), upperInputName.begin(), safeToUpper); bool isStdIn = upperInputName == "STDIN"; // In mode "info", we want to display the information in the stream header only. // We can reuse the existing code but we need to: // create an InfoPrinter with a dedicated INFO type // disable logging outside of this printer (=> _verbosity=0) // decompress no block (=> _outputName = NONE and --from=1 and --to=1) // disable threading for proper display (=> _jobs=1) bool isInfo = _ctx.getString("mode") == "y"; int vl = _verbosity; if (isInfo) { _verbosity = 0; _jobs = 1; _outputName = "NONE"; _ctx.putString("outputName", _outputName); _ctx.putInt("from", 1); _ctx.putInt("to", 1); _ctx.putInt("jobs", 1); if (isStdIn == true) { cerr << "STDIN not allowed in 'info' mode" << endl; return Error::ERR_OPEN_FILE; } } if (isStdIn == false) { vector errors; bool isRecursive = (_inputName.length() < 2) || (_inputName[_inputName.length() - 2] != PATH_SEPARATOR) || (_inputName[_inputName.length() - 1] != '.'); FileListConfig cfg = { isRecursive, _noLinks, false, _noDotFiles }; createFileList(_inputName, files, cfg, errors); if (errors.size() > 0) { for (size_t i = 0; i < errors.size(); i++) cerr << errors[i] << endl; return Error::ERR_OPEN_FILE; } if (files.size() == 0) { cerr << "Cannot find any file to decompress" << endl; return Error::ERR_OPEN_FILE; } nbFiles = int(files.size()); string strFiles = nbFiles > 1 ? " files" : " file"; ss << nbFiles << strFiles << " to decompress\n"; log.println(ss.str(), _verbosity > 0); ss.str(string()); } string upperOutputName = _outputName; transform(upperOutputName.begin(), upperOutputName.end(), upperOutputName.begin(), safeToUpper); bool isStdOut = upperOutputName == "STDOUT"; // Limit verbosity level when output is stdout // Logic is duplicated here to avoid dependency to Kanzi.cpp if (isStdOut == true) _verbosity = 0; // Limit verbosity level when files are processed concurrently if ((_verbosity > 1) && (_jobs > 1) && (nbFiles > 1)) { log.println("Warning: limiting verbosity to 1 due to concurrent processing of input files.\n", true); _verbosity = 1; } if (_verbosity > 2) { ss << "Verbosity: " << _verbosity << endl; ss << "Overwrite: " << (_overwrite ? "true" : "false") << endl; ss << "Using " << _jobs << " job" << (_jobs > 1 ? "s" : "") << endl; log.print(ss.str(), true); ss.str(string()); } const int from = _ctx.getInt("from", 1); InfoPrinter::Type ipt = isInfo ? InfoPrinter::INFO : InfoPrinter::DECOMPRESSION; InfoPrinter listener(vl, ipt, cout, max(from, 1)); if ((vl > 2) || ((isInfo == true) && (vl > 0))) addListener(listener); int res = 0; bool inputIsDir = false; string formattedOutName = _outputName; string formattedInName = _inputName; bool specialOutput = (isStdOut == true) || (upperOutputName == "NONE"); // Need to strip path separator at the end to make 'stat()' happy if ((formattedOutName.size() > 1) && (formattedOutName[formattedOutName.size() - 1] == PATH_SEPARATOR)) { formattedOutName.resize(formattedOutName.size() - 1); } if (isStdIn == false) { struct STAT buffer; if ((formattedInName.size() > 1) && (formattedInName[formattedInName.size() - 1] == PATH_SEPARATOR)) { formattedInName.resize(formattedInName.size() - 1); } if (STAT(formattedInName.c_str(), &buffer) != 0) { cerr << "Cannot access input file '" << formattedInName << "'" << endl; return Error::ERR_OPEN_FILE; } if ((buffer.st_mode & S_IFDIR) != 0) { inputIsDir = true; if ((formattedInName.size() != 0) && (formattedInName[formattedInName.size() - 1] == '.')) { formattedInName.resize(formattedInName.size() - 1); } if ((formattedInName.size() != 0) && (formattedInName[formattedInName.size() - 1] != PATH_SEPARATOR)) { formattedInName += PATH_SEPARATOR; } if ((formattedOutName.size() != 0) && (specialOutput == false)) { if (STAT(formattedOutName.c_str(), &buffer) != 0) { cerr << "Output must be an existing directory (or 'NONE')" << endl; return Error::ERR_OPEN_FILE; } if ((buffer.st_mode & S_IFDIR) == 0) { cerr << "Output must be a directory (or 'NONE')" << endl; return Error::ERR_CREATE_FILE; } formattedOutName += PATH_SEPARATOR; } } else { inputIsDir = false; if ((formattedOutName.size() != 0) && (specialOutput == false)) { if ((STAT(formattedOutName.c_str(), &buffer) == 0) && ((buffer.st_mode & S_IFDIR) != 0)) { cerr << "Output must be a file (or 'NONE')" << endl; return Error::ERR_CREATE_FILE; } } } } _ctx.putInt("verbosity", _verbosity); #ifdef CONCURRENCY_ENABLED ThreadPool pool(_jobs + 1); // +1 to avoid deadlock due to thread exhaustion #endif // Run the task(s) if (nbFiles == 1) { string oName = formattedOutName; string iName = "STDIN"; if (isStdIn == true) { if (oName.length() == 0) { oName = "STDOUT"; } } else { iName = files[0].fullPath(); _ctx.putLong("fileSize", files[0]._size); string upperIName = iName; transform(upperIName.begin(), upperIName.end(), upperIName.begin(), safeToUpper); if (oName.length() == 0) { oName = iName; if ((upperIName.length() >= 4) && (upperIName.substr(upperIName.length() - 4) == ".KNZ")) oName.resize(oName.length() - 4); else oName = oName + ".bak"; } else if ((inputIsDir == true) && (specialOutput == false)) { oName = formattedOutName + iName.substr(formattedInName.size()); if ((upperIName.length() >= 4) && (upperIName.substr(upperIName.length() - 4) == ".KNZ")) oName.resize(oName.length() - 4); else oName = oName + ".bak"; } } _ctx.putString("inputName", iName); _ctx.putString("outputName", oName); FileDecompressTask task(_ctx, _listeners); FileDecompressResult fdr = task.run(); res = fdr._code; read = fdr._read; if (res != 0) { cerr << fdr._errMsg << endl; } } else { vector*> tasks; tasks.reserve(nbFiles); #ifdef CONCURRENCY_ENABLED vector jobsPerTask(nbFiles); vector*> workers; Global::computeJobsPerTask(jobsPerTask.data(), _jobs, nbFiles); #endif try { sortFilesByPathAndSize(files, true); // Create one task per file for (int i = 0; i < nbFiles; i++) { string oName = formattedOutName; string iName = files[i].fullPath(); upperInputName = iName; transform(upperInputName.begin(), upperInputName.end(), upperInputName.begin(), safeToUpper); if (oName.length() == 0) { oName = iName; if ((upperInputName.length() >= 4) && (upperInputName.substr(upperInputName.length() - 4) == ".KNZ")) oName.resize(oName.length() - 4); else oName = oName + ".bak"; } else if ((inputIsDir == true) && (specialOutput == false)) { oName = formattedOutName + iName.substr(formattedInName.size()); if ((upperInputName.length() >= 4) && (upperInputName.substr(upperInputName.length() - 4) == ".KNZ")) oName.resize(oName.length() - 4); else oName = oName + ".bak"; } #ifdef CONCURRENCY_ENABLED Context taskCtx(_ctx, &pool); taskCtx.putInt("jobs", jobsPerTask[i]); #else Context taskCtx(_ctx); taskCtx.putInt("jobs", 1); #endif taskCtx.putLong("fileSize", files[i]._size); taskCtx.putString("inputName", iName); taskCtx.putString("outputName", oName); tasks.push_back(new FileDecompressTask(taskCtx, _listeners)); } bool doConcurrent = _jobs > 1; #ifdef CONCURRENCY_ENABLED if (doConcurrent) { BoundedConcurrentQueue queue(nbFiles, &tasks[0]); vector > results; workers.reserve(_jobs); results.reserve(_jobs); try { // Create one worker per job and run it. A worker calls several tasks sequentially. for (int i = 0; i < _jobs; i++) { workers.push_back(new FileDecompressWorker*, FileDecompressResult>(&queue)); if (_ctx.getPool() == nullptr) results.push_back(std::async(launch::async, &FileDecompressWorker::run, workers[i])); else results.push_back(_ctx.getPool()->schedule(&FileDecompressWorker::run, workers[i])); } // Wait for results for (int i = 0; i < _jobs; i++) { FileDecompressResult fdr = results[i].get(); read += fdr._read; if (fdr._code != 0) { if (res == 0) res = fdr._code; cerr << fdr._errMsg << endl; // Exit early by telling the workers that the queue is empty queue.clear(); } } } catch (...) { queue.clear(); for (uint i = 0; i < results.size(); i++) { try { if (results[i].valid()) results[i].wait(); } catch (const exception&) { } } throw; } } #endif if (!doConcurrent) { for (uint i = 0; i < tasks.size(); i++) { FileDecompressResult fdr = tasks[i]->run(); res = fdr._code; read += fdr._read; if (res != 0) { cerr << fdr._errMsg << endl; break; } } } } catch (...) { #ifdef CONCURRENCY_ENABLED for (uint i = 0; i < workers.size(); i++) delete workers[i]; #endif for (uint i = 0; i < tasks.size(); i++) delete tasks[i]; throw; } #ifdef CONCURRENCY_ENABLED for (uint i = 0; i < workers.size(); i++) delete workers[i]; #endif for (uint i = 0; i < tasks.size(); i++) delete tasks[i]; } stopClock.stop(); if ((nbFiles > 1) && (_verbosity > 0)) { double delta = stopClock.elapsed(); log.println("", true); ss << "Total decompression time: "; if (delta >= 1e5) { ss.precision(1); ss.setf(ios::fixed); ss << (delta / 1000) << " s" << endl; } else { ss << int(delta) << " ms" << endl; } ss << "Total output size: " << read << (read > 1 ? " bytes" : " byte") << endl; log.print(ss.str(), true); ss.str(string()); } if (_verbosity > 2) removeListener(listener); inputSize += read; return res; } bool BlockDecompressor::addListener(Listener& bl) { _listeners.push_back(&bl); return true; } bool BlockDecompressor::removeListener(Listener& bl) { std::vector*>::iterator it = find(_listeners.begin(), _listeners.end(), &bl); if (it == _listeners.end()) return false; _listeners.erase(it); return true; } void BlockDecompressor::notifyListeners(vector*>& listeners, const Event& evt) { for (size_t i = 0; i < listeners.size(); i++) listeners[i]->processEvent(evt); } template FileDecompressTask::FileDecompressTask(const Context& ctx, vector*>& listeners) : _ctx(ctx) , _listeners(listeners) { _os = nullptr; _cis = nullptr; } template FileDecompressTask::~FileDecompressTask() { dispose(); if (_cis != nullptr) { delete _cis; _cis = nullptr; } try { if ((_os != nullptr) && (_os != &cout)) { delete _os; } _os = nullptr; } catch (const exception&) { // Ignore: best effort } } template T FileDecompressTask::run() { Printer log(cout); int verbosity = _ctx.getInt("verbosity"); string inputName = _ctx.getString("inputName"); string outputName = _ctx.getString("outputName"); stringstream ss; if (verbosity > 2) { ss << "Input file name: '" << inputName << "'" << endl; ss << "Output file name: '" << outputName << "'" << endl; log.print(ss.str(), true); ss.str(string()); } bool overwrite = _ctx.getInt("overwrite") != 0; int64 read = 0; ss << "\nDecompressing " << inputName << " ..."; log.println(ss.str(), verbosity > 1); log.println("\n", verbosity > 3); WallTimer timer; if (_listeners.size() > 0) { Event evt(Event::DECOMPRESSION_START, 0, int64(0), timer.getCurrentTime()); BlockDecompressor::notifyListeners(_listeners, evt); } string str = outputName; transform(str.begin(), str.end(), str.begin(), safeToUpper); ofstream* fos = nullptr; #if defined(WIN32) || defined(_WIN32) || defined(_WIN64) bool checkOutputSize = str != "NUL"; #else bool checkOutputSize = str != "/DEV/NULL"; #endif #define CLEANUP_DECOMP_OS if ((_os != nullptr) && (_os != &cout)) { \ delete _os; \ _os = nullptr; \ } if (str == "NONE") { _os = new NullOutputStream(); checkOutputSize = false; } else if (str == "STDOUT") { _os = &cout; checkOutputSize = false; } else { try { if (samePaths(inputName, outputName)) { stringstream sserr; sserr << "The input and output files must be different"; return T(Error::ERR_CREATE_FILE, 0, sserr.str()); } struct STAT buffer; if (STAT(outputName.c_str(), &buffer) == 0) { if ((buffer.st_mode & S_IFDIR) != 0) { stringstream sserr; sserr << "The output file is a directory"; return T(Error::ERR_OUTPUT_IS_DIR, 0, sserr.str()); } if (overwrite == false) { stringstream sserr; sserr << "File '" << outputName << "' exists and the 'force' command " << "line option has not been provided"; return T(Error::ERR_OVERWRITE_FILE, 0, sserr.str()); } // Delete output file to ensure consistent performance remove(outputName.c_str()); } ofstream* ofs = new ofstream(outputName.c_str(), ofstream::out | ofstream::binary); if (!*ofs) { string errMsg; if (overwrite == true) { // Attempt to create the full folder hierarchy to file string parentDir = outputName; size_t idx = outputName.find_last_of(PATH_SEPARATOR); if (idx != string::npos) parentDir.resize(idx); int rmkd = mkdirAll(parentDir); if ((rmkd == 0) || (rmkd == EEXIST)) { delete ofs; ofs = new ofstream(outputName.c_str(), ofstream::binary); } else { errMsg = strerror(rmkd); } } if (!*ofs) { delete ofs; stringstream sserr; sserr << "Cannot open output file '" << outputName << "' for writing"; if (errMsg != "") sserr << ": " << errMsg; return T(Error::ERR_CREATE_FILE, 0, sserr.str()); } } _os = ofs; fos = ofs; } catch (const exception& e) { stringstream sserr; sserr << "Cannot open output file '" << outputName << "' for writing: " << e.what(); return T(Error::ERR_CREATE_FILE, 0, sserr.str()); } } InputStream* is = nullptr; #define CLEANUP_DECOMP_IS if ((is != nullptr) && (is != &cin)) { \ delete is; \ is = nullptr; \ } try { str = inputName; transform(str.begin(), str.end(), str.begin(), safeToUpper); if (str == "STDIN") { is = &cin; } else { ifstream* ifs = new ifstream(inputName.c_str(), ifstream::in | ifstream::binary); if (!*ifs) { delete ifs; CLEANUP_DECOMP_OS stringstream sserr; sserr << "Cannot open input file '" << inputName << "'"; return T(Error::ERR_OPEN_FILE, 0, sserr.str()); } is = ifs; } try { _cis = new CompressedInputStream(*is, _ctx); for (uint i = 0; i < _listeners.size(); i++) _cis->addListener(*_listeners[i]); } catch (const invalid_argument& e) { CLEANUP_DECOMP_IS CLEANUP_DECOMP_OS stringstream sserr; sserr << "Cannot create compressed stream: " << e.what(); return T(Error::ERR_CREATE_DECOMPRESSOR, 0, sserr.str()); } } catch (const exception& e) { CLEANUP_DECOMP_IS CLEANUP_DECOMP_OS stringstream sserr; sserr << "Cannot open input file '" << inputName << "': " << e.what(); const uint64 readSoFar = (_cis != nullptr) ? _cis->getRead() : 0; return T(Error::ERR_OPEN_FILE, readSoFar, sserr.str()); } Clock stopClock; kanzi::byte* buf = new kanzi::byte[DEFAULT_BUFFER_SIZE]; #define CLEANUP_DECOMP_RESOURCES(readVar) \ dispose(); \ const uint64 readVar = _cis->getRead(); \ delete _cis; \ _cis = nullptr; \ CLEANUP_DECOMP_IS \ CLEANUP_DECOMP_OS \ delete[] buf #define CLEANUP_DECOMP_RESOURCES_EOF(readVar, eofVar) \ dispose(); \ const uint64 readVar = _cis->getRead(); \ const bool eofVar = _cis->eof(); \ delete _cis; \ _cis = nullptr; \ CLEANUP_DECOMP_IS \ CLEANUP_DECOMP_OS \ delete[] buf try { SliceArray sa(buf, DEFAULT_BUFFER_SIZE, 0); int decoded = 0; // Decode next block do { _cis->read(reinterpret_cast(&sa._array[0]), sa._length); decoded = int(_cis->gcount()); if (decoded < 0) { CLEANUP_DECOMP_RESOURCES(d); stringstream sserr; sserr << "Reached end of stream"; return T(Error::ERR_READ_FILE, d, sserr.str()); } try { if (decoded > 0) { _os->write(reinterpret_cast(&sa._array[0]), decoded); if (_os->fail() || _os->bad()) { CLEANUP_DECOMP_RESOURCES(d); stringstream sserr; sserr << "Failed to write decompressed block to file '" << outputName << "'"; return T(Error::ERR_WRITE_FILE, d, sserr.str()); } read += decoded; } } catch (const exception& e) { CLEANUP_DECOMP_RESOURCES(d); stringstream sserr; sserr << "Failed to write decompressed block to file '" << outputName << "': " << e.what(); return T(Error::ERR_WRITE_FILE, d, sserr.str()); } } while (_cis->eof() == 0); } catch (const IOException& e) { CLEANUP_DECOMP_RESOURCES_EOF(d, isEOF); if (isEOF == true) return T(Error::ERR_READ_FILE, d, "Reached end of stream"); stringstream sserr; sserr << e.what(); return T(e.error(), d, sserr.str()); } catch (const exception& e) { CLEANUP_DECOMP_RESOURCES_EOF(d, isEOF); if (isEOF == true) return T(Error::ERR_READ_FILE, d, "Reached end of stream"); stringstream sserr; sserr << "An unexpected condition happened. Exiting ..." << endl << e.what(); return T(Error::ERR_UNKNOWN, d, sserr.str()); } uint64 written = 0; try { _os->flush(); if (_os->fail() || _os->bad()) { CLEANUP_DECOMP_RESOURCES(d); stringstream sserr; sserr << "Failed to flush decompressed output file '" << outputName << "'"; return T(Error::ERR_WRITE_FILE, d, sserr.str()); } if (checkOutputSize == true) { const streampos pos = _os->tellp(); if ((pos == streampos(-1)) || _os->fail() || _os->bad()) { CLEANUP_DECOMP_RESOURCES(d); stringstream sserr; sserr << "Failed to query decompressed output file '" << outputName << "'"; return T(Error::ERR_WRITE_FILE, d, sserr.str()); } written = uint64(pos); } if (fos != nullptr) { fos->close(); if (!*fos) { CLEANUP_DECOMP_RESOURCES(d); stringstream sserr; sserr << "Failed to close decompressed output file '" << outputName << "'"; return T(Error::ERR_WRITE_FILE, d, sserr.str()); } } } catch (const exception& e) { CLEANUP_DECOMP_RESOURCES(d); stringstream sserr; sserr << "Failed to finalize decompressed output file '" << outputName << "': " << e.what(); return T(Error::ERR_WRITE_FILE, d, sserr.str()); } // Close input stream before deleting resources dispose(); const uint64 decoded = _cis->getRead(); // Clean up resources at the end of the method as the task may be // recycled in a threadpool and the destructor not called. delete _cis; _cis = nullptr; // is destructor will call close if ifstream CLEANUP_DECOMP_IS CLEANUP_DECOMP_OS // If the whole input stream has been decoded and the original data size is present, // check that the output size matches the original data size. if ((checkOutputSize == true) && (_ctx.has("to") == false) && (_ctx.has("from") == false)) { const uint64 outputSize = _ctx.getLong("outputSize", 0); if ((outputSize != 0) && (written != outputSize)) { delete[] buf; stringstream sserr; sserr << "Corrupted bitstream: invalid output size (expected " << outputSize; sserr << ", got " << written << ")"; return T(Error::ERR_INVALID_FILE, decoded, sserr.str()); } } stopClock.stop(); double delta = stopClock.elapsed(); if (verbosity >= 1) { log.println("", verbosity > 1); ss.str(string()); if (verbosity > 1) { if (delta >= 1e5) { ss.precision(1); ss.setf(ios::fixed); ss << "Decompression time: " << (delta / 1000) << " s" << endl; } else { ss << "Decompression time: " << int(delta) << " ms" << endl; } ss << "Input size: " << decoded << endl; ss << "Output size: " << read << endl; log.print(ss.str(), true); ss.str(string()); } if (verbosity == 1) { ss << "Decompressed " << inputName << ": " << decoded << " => " << read; if (delta >= 1e5) { ss.precision(1); ss.setf(ios::fixed); ss << " bytes in " << (delta / 1000) << " s"; } else { ss << " bytes in " << int(delta) << " ms"; } log.println(ss.str(), true); ss.str(string()); } if ((verbosity > 1) && (delta > 0)) { double b2KiB = double(1000) / double(1024); ss << "Throughput (KiB/s): " << uint(double(read) * b2KiB / delta); log.println(ss.str(), true); ss.str(string()); } log.println("", verbosity > 1); } if (_listeners.size() > 0) { Event evt(Event::DECOMPRESSION_END, 0, int64(decoded), timer.getCurrentTime()); BlockDecompressor::notifyListeners(_listeners, evt); } if (_ctx.getInt("remove", 0) != 0) { // Delete input file if (inputName == "STDIN") { log.println("Warning: ignoring remove option with STDIN", verbosity > 0); } else if (remove(inputName.c_str()) != 0) { log.println("Warning: input file could not be deleted", verbosity > 0); } } #undef CLEANUP_DECOMP_RESOURCES #undef CLEANUP_DECOMP_RESOURCES_EOF #undef CLEANUP_DECOMP_IS #undef CLEANUP_DECOMP_OS delete[] buf; return T(0, read, ""); } // Close input stream. Do not deallocate resources. Idempotent. template void FileDecompressTask::dispose() { try { if (_cis != nullptr) { _cis->close(); } } catch (const exception& e) { cerr << "Decompression failure: " << e.what() << endl; } // _os destructor will call close if ofstream } #ifdef CONCURRENCY_ENABLED template R FileDecompressWorker::run() { int res = 0; uint64 read = 0; string errMsg; while (res == 0) { T* task = _queue->get(); if (task == nullptr) break; R result = (*task)->run(); res = result._code; read += result._read; if (res != 0) { errMsg += result._errMsg; } } return R(res, read, errMsg); } #endif kanzi-cpp-2.6.0/src/app/BlockDecompressor.hpp000066400000000000000000000071201525576310500211240ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_BlockDecompressor #define knz_BlockDecompressor #include #include #include "../OutputStream.hpp" #include "../io/CompressedInputStream.hpp" namespace kanzi { class FileDecompressResult { public: int _code; uint64 _read; std::string _errMsg; FileDecompressResult() : _code(0) , _read(0) , _errMsg() { } FileDecompressResult(int code, uint64 read, const std::string& errMsg) : _code(code) , _read(read) , _errMsg(errMsg) { } #if __cplusplus < 201103L FileDecompressResult(const FileDecompressResult& fdr) : _code(fdr._code) , _read(fdr._read) , _errMsg(fdr._errMsg) { } FileDecompressResult& operator=(const FileDecompressResult& fdr) { _errMsg = fdr._errMsg; _code = fdr._code; _read = fdr._read; return *this; } ~FileDecompressResult() {} #else FileDecompressResult(const FileDecompressResult& fcr) = delete; FileDecompressResult& operator=(const FileDecompressResult& fcr) = delete; FileDecompressResult(FileDecompressResult&& fcr) = default; FileDecompressResult& operator=(FileDecompressResult&& fcr) = default; ~FileDecompressResult() = default; #endif }; #ifdef CONCURRENCY_ENABLED template class FileDecompressWorker FINAL : public Task { public: FileDecompressWorker(BoundedConcurrentQueue* queue) : _queue(queue) { } ~FileDecompressWorker() {} R run(); private: BoundedConcurrentQueue* _queue; }; #endif template class FileDecompressTask FINAL : public Task { public: static const int DEFAULT_BUFFER_SIZE = 65536; FileDecompressTask(const Context& ctx, std::vector*>& listeners); ~FileDecompressTask(); T run(); void dispose(); private: Context _ctx; OutputStream* _os; CompressedInputStream* _cis; std::vector*> _listeners; }; typedef FileDecompressTask FDTask; class BlockDecompressor { friend class FileDecompressTask; public: BlockDecompressor(const Context& ctx); ~BlockDecompressor(); int decompress(uint64& read); // Register a copy of the listener // Not thread safe bool addListener(Listener& bl); // Not thread safe bool removeListener(Listener& bl); void dispose() const {}; private: int _verbosity; bool _overwrite; std::string _inputName; std::string _outputName; int _blockSize; int _jobs; std::vector*> _listeners; bool _noDotFiles; bool _noLinks; Context _ctx; static void notifyListeners(std::vector*>& listeners, const Event& evt); }; } #endif kanzi-cpp-2.6.0/src/app/InfoPrinter.cpp000066400000000000000000000244431525576310500177450ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include "InfoPrinter.hpp" #include "../util/strings.hpp" using namespace kanzi; using namespace std; InfoPrinter::InfoPrinter(int infoLevel, InfoPrinter::Type type, OutputStream& os, int firstBlockId) : _os(os) , _type(type) , _level(infoLevel) , _headerInfo(0) { STORE_ATOMIC(_nextBlockId, firstBlockId); if (type == InfoPrinter::COMPRESSION) { _thresholds[0] = Event::COMPRESSION_START; _thresholds[1] = Event::BEFORE_TRANSFORM; _thresholds[2] = Event::AFTER_TRANSFORM; _thresholds[3] = Event::BEFORE_ENTROPY; _thresholds[4] = Event::AFTER_ENTROPY; _thresholds[5] = Event::COMPRESSION_END; } else { _thresholds[0] = Event::DECOMPRESSION_START; _thresholds[1] = Event::BEFORE_ENTROPY; _thresholds[2] = Event::AFTER_ENTROPY; _thresholds[3] = Event::BEFORE_TRANSFORM; _thresholds[4] = Event::AFTER_TRANSFORM; _thresholds[5] = Event::DECOMPRESSION_END; } } void InfoPrinter::processEvent(const Event& evt) { if (_type == InfoPrinter::INFO) { processHeaderInfo(evt); return; } #ifdef CONCURRENCY_ENABLED Event::Type t = evt.getType(); if (t == Event::BEFORE_TRANSFORM || t == Event::AFTER_TRANSFORM || t == Event::BEFORE_ENTROPY || t == Event::AFTER_ENTROPY) { processBlockEventOrdered(evt); return; } #endif processEventOrdered(evt); } #ifdef CONCURRENCY_ENABLED void InfoPrinter::processBlockEventOrdered(const Event& evt) { const int blockId = evt.getId(); const Event::Type type = evt.getType(); bool blockComplete = false; // Determine completion condition if (_type == InfoPrinter::COMPRESSION) { blockComplete = (type == Event::AFTER_ENTROPY); } else if (_type == InfoPrinter::DECOMPRESSION) { blockComplete = (type == Event::AFTER_TRANSFORM); } { #ifdef CONCURRENCY_ENABLED std::lock_guard lock(_mutex); #endif _pendingBlocks[blockId].push_back(evt); // Do not attempt to release unless this block is complete if (blockComplete == false) return; } // Try to release completed blocks in strict blockId order for (;;) { std::vector events; { #ifdef CONCURRENCY_ENABLED std::lock_guard lock(_mutex); #endif int expectedId = LOAD_ATOMIC(_nextBlockId); std::map >::iterator it = _pendingBlocks.find(expectedId); if (it == _pendingBlocks.end()) return; // The block must be complete before release bool complete = false; const std::vector& evts = it->second; for (size_t i = 0; i < evts.size(); i++) { if ((_type == InfoPrinter::COMPRESSION) && (evts[i].getType() == Event::AFTER_ENTROPY)) { complete = true; break; } if ((_type == InfoPrinter::DECOMPRESSION) && (evts[i].getType() == Event::AFTER_TRANSFORM)) { complete = true; break; } } if (complete == false) return; // Release block events.swap(it->second); _pendingBlocks.erase(it); STORE_ATOMIC(_nextBlockId, expectedId + 1); } // Process all events for this block in arrival order #ifdef CONCURRENCY_ENABLED std::lock_guard lock(_mutex); #endif for (size_t i = 0; i < events.size(); i++) { processEventOrdered(events[i]); } } } #endif void InfoPrinter::processEventOrdered(const Event& evt) { const int blockId = evt.getId(); const Event::Type type = evt.getType(); string msg; if (type == _thresholds[1]) { BlockInfo* bi = new BlockInfo(); bi->_timeStamp1 = evt.getTime(); bi->_stage0Size = evt.getSize(); _blocks[blockId] = bi; if (_level >= 5) { msg = evt.toString(); } } else if (type == _thresholds[2]) { std::map::iterator it = _blocks.find(blockId); if (it == _blocks.end()) return; BlockInfo& bi = *it->second; bi._timeStamp2 = evt.getTime(); if (_level >= 5) { double elapsed = WallTimer::calculateDifference(bi._timeStamp1, bi._timeStamp2); std::stringstream ss; ss << evt.toString() << " [" << int64(elapsed) << " ms]"; msg = ss.str(); } } else if (type == _thresholds[3]) { std::map::iterator it = _blocks.find(blockId); if (it == _blocks.end()) return; BlockInfo& bi = *it->second; bi._timeStamp3 = evt.getTime(); bi._stage1Size = evt.getSize(); if (_level >= 5) msg = evt.toString(); } else if (type == _thresholds[4]) { std::map::iterator it = _blocks.find(blockId); if (it == _blocks.end()) return; const BlockInfo& bi = *it->second; stringstream ss; if (_level >= 5) { ss << evt.toString() << endl; } // Display block info if (_level >= 4) { double elapsed1 = WallTimer::calculateDifference(bi._timeStamp1, bi._timeStamp2); double elapsed2 = WallTimer::calculateDifference(bi._timeStamp3, evt.getTime()); ss << "Block " << blockId << ": " << bi._stage0Size << " => " << bi._stage1Size << " [" << int64(elapsed1) << " ms] => " << evt.getSize() << " [" << int64(elapsed2) << " ms]"; // Add compression ratio for encoding if ((_type == InfoPrinter::COMPRESSION) && (bi._stage0Size != 0)) { ss << " (" << uint(double(evt.getSize()) * 100.0 / double(bi._stage0Size)) << "%)"; } // Optionally add hash if (evt.getHash() != 0) { ss << std::uppercase << std::hex << " [" << evt.getHash() << "]"; } msg = ss.str(); } delete it->second; _blocks.erase(it); } else if ((evt.getType() == Event::AFTER_HEADER_DECODING) && (_level >= 3)) { Event::HeaderInfo* info = evt.getInfo(); if (info == nullptr) return; if (_level >= 5) { // JSON output msg = evt.toString(); } else { // Raw text output stringstream ss; ss << "Bitstream version: " << info->bsVersion << endl; string strCk = "NONE"; if (info->checksumSize == 32) strCk = "32 bits"; else if (info->checksumSize == 64) strCk = "64 bits"; ss << "Block checksum: " << strCk << endl; ss << "Block size: " << info->blockSize << " bytes" << endl; string strE = info->entropyType == "NONE" ? "no" : info->entropyType; ss << "Using " << strE << " entropy codec (stage 1)" << endl; string strF = info->transformType == "NONE" ? "no" : info->transformType; ss << "Using " << strF << " transform (stage 2)" << endl; if (info->originalSize >= 0) ss << "Original size: " << info->originalSize << " byte(s)" << endl; msg = ss.str(); } } else if (_level >= 5) { msg = evt.toString(); } if (msg.size() > 0) { _os << msg << endl; _os.flush(); } } void InfoPrinter::processHeaderInfo(const Event& evt) { if ((_level == 0) || (evt.getType() != Event::AFTER_HEADER_DECODING)) return; const Event::HeaderInfo* info = evt.getInfo(); if (info == nullptr) return; stringstream ss; if (_headerInfo++ == 0) { ss << endl; ss << "|" << " File Name "; ss << "|" << "Ver"; ss << "|" << "Check"; ss << "|" << "Block Size"; ss << "|" << " File Size "; ss << "|" << " Orig. Size "; ss << "|" << " Ratio "; if (_level >= 4) { ss << "|" << " Entropy"; ss << "|" << " Transforms "; } ss << "|" << endl; } ss << "|"; string inputName = info->inputName; size_t idx = inputName.find_last_of(PATH_SEPARATOR); if (idx != string::npos) inputName.erase(0, idx + 1); if (inputName.length() > 20) inputName.replace(18, string::npos, ".."); ss << left << setw(20) << inputName << "|" << right; ss << setw(3) << info->bsVersion << "|"; ss << setw(5) << info->checksumSize << "|"; ss << setw(10) << info->blockSize << "|"; if (info->fileSize >= 0) ss << setw(12) << formatSize(double(info->fileSize)) << "|"; else ss << setw(12) << " N/A |"; if (info->originalSize >= 0) ss << setw(12) << formatSize(double(info->originalSize)) << "|"; else ss << setw(12) << " N/A |"; if ((info->originalSize >= 0) && (info->fileSize >= 0)) { double compSz = double(info->fileSize); double origSz = double(info->originalSize); if (origSz == 0.0) ss << setw(7) << " N/A |"; else ss << setw(7) << fixed << setprecision(3) << (compSz / origSz) << "|"; } else { ss << setw(7) << " N/A |"; } if (_level >= 4) { ss << setw(8) << info->entropyType << "|"; string t = info->transformType; if (t.length() > 26) t.replace(24, string::npos, ".."); ss << setw(26) << t << "|"; } _os << ss.str() << endl; } kanzi-cpp-2.6.0/src/app/InfoPrinter.hpp000066400000000000000000000042701525576310500177460ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_InfoPrinter #define knz_InfoPrinter #include #include #ifdef CONCURRENCY_ENABLED #include #endif #include #include #include "../concurrent.hpp" #include "../Event.hpp" #include "../Listener.hpp" #include "../OutputStream.hpp" #include "../util/Clock.hpp" namespace kanzi { class BlockInfo { public: int64 _stage0Size; int64 _stage1Size; WallTimer::TimeData _timeStamp1; WallTimer::TimeData _timeStamp2; WallTimer::TimeData _timeStamp3; BlockInfo() : _stage0Size(0), _stage1Size(0) {} }; class InfoPrinter : public Listener { public: enum Type { COMPRESSION, DECOMPRESSION, INFO }; InfoPrinter(int infoLevel, InfoPrinter::Type type, OutputStream& os, int firstBlockId = 1); ~InfoPrinter() {} void processEvent(const Event& evt); private: #ifdef CONCURRENCY_ENABLED // Ordered-phase handling void processBlockEventOrdered(const Event& evt); #endif // Actual event processing + printing void processEventOrdered(const Event& evt); // Header-only info void processHeaderInfo(const Event& evt); OutputStream& _os; InfoPrinter::Type _type; int _level; int _headerInfo; // Per-block state std::map _blocks; Event::Type _thresholds[6]; #ifdef CONCURRENCY_ENABLED std::mutex _mutex; #endif std::map > _pendingBlocks; atomic_int_t _nextBlockId; }; } #endif kanzi-cpp-2.6.0/src/app/Kanzi.cpp000066400000000000000000001213051525576310500165550ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include #include "BlockCompressor.hpp" #include "BlockDecompressor.hpp" #include "../Error.hpp" #include "../util/Printer.hpp" #if defined(WIN32) || defined(_WIN32) || defined(_WIN64) #include #include #include #endif using namespace kanzi; using namespace std; static const string CMD_LINE_ARGS[15] = { "-c", "-d", "-i", "-o", "-b", "-t", "-e", "-j", "-v", "-l", "-s", "-x", "-f", "-h", "-y" }; //static const int ARG_IDX_COMPRESS = 0; //static const int ARG_IDX_DECOMPRESS = 1; static const int ARG_IDX_INPUT = 2; static const int ARG_IDX_OUTPUT = 3; static const int ARG_IDX_BLOCK = 4; static const int ARG_IDX_TRANSFORM = 5; static const int ARG_IDX_ENTROPY = 6; static const int ARG_IDX_JOBS = 7; static const int ARG_IDX_VERBOSE = 8; static const int ARG_IDX_LEVEL = 9; //static const int ARG_IDX_CHECKSUM = 10; //static const int ARG_IDX_FROM = 11; //static const int ARG_IDX_TO = 12; static const string KANZI_VERSION = "2.6.0"; static const string APP_HEADER = "Kanzi " + KANZI_VERSION + " (c) Frederic Langlet"; static const string APP_SUB_HEADER = "Fast lossless data compressor."; static const string APP_USAGE = "Usage: kanzi [-c|-d|-y] [flags and files in any order]"; #ifdef CONCURRENCY_ENABLED static const int MAX_CONCURRENCY = 64; #endif void printHelp(Printer& log, const string& mode, bool showHeader) { log.println("", true); if (showHeader == true) { log.println(APP_HEADER, true); log.println("", true); log.println(APP_SUB_HEADER, true); } log.println(APP_USAGE, true); log.println("", true); log.println("Options\n", true); log.println(" -h, --help", true); if ((mode != "c") && (mode != "d") && (mode != "y")) { log.println(" Display this message.", true); log.println(" Use in conjunction with -c to print information for compression,", true); log.println(" or -d to print information for decompression.\n", true); log.println(" -c, --compress", true); log.println(" Compress mode\n", true); log.println(" -d, --decompress", true); log.println(" Decompress mode\n", true); log.println(" -y, --info", true); log.println(" Info mode: display information about compressed files\n", true); } else { log.println(" Display this message.\n", true); } log.println(" -i, --input=", true); log.println(" Name of the input file or directory or 'stdin'", true); log.println(" When the source is a directory, all files in it will be processed.", true); #if defined(WIN32) || defined(_WIN32) || defined(_WIN64) log.println(" Provide \\. at the end of the directory name to avoid recursion", true); log.println(" (EG: myDir\\. => no recursion)", true); #else log.println(" Provide /. at the end of the directory name to avoid recursion", true); log.println(" (EG: myDir/. => no recursion)", true); #endif log.println(" If this option is not provided, kanzi reads data from stdin.\n", true); if (mode != "y") { log.println(" -o, --output=", true); if (mode == "c") { log.println(" Optional name of the output file or directory (defaults to", true); log.println(" if input is or 'stdout' if input is 'stdin').", true); log.println(" or 'none' or 'stdout'.\n", true); } else if (mode == "d") { log.println(" Optional name of the output file or directory (defaults to", true); log.println(" if input is or 'stdout' if input is 'stdin').", true); log.println(" or 'none' or 'stdout'.\n", true); } else { log.println(" Optional name of the output file or 'none' or 'stdout'.\n", true); } } if (mode == "c") { log.println(" -b, --block=", true); log.println(" Size of blocks (default 4|8|16|32 MiB based on level, max 1 GiB, min 1 KiB).", true); log.println(" 'auto' means that the compressor derives the best value", true); log.println(" based on input size (when available) and number of jobs.\n", true); log.println(" -l, --level=", true); log.println(" Set the compression level [0..9]", true); log.println(" Providing this option forces entropy and transform.", true); log.println(" Defaults to level 3 if not provided.\n", true); log.println(" 0 = NONE&NONE (store)", true); log.println(" 1 = LZX&NONE", true); log.println(" 2 = DNA+LZ&HUFFMAN", true); log.println(" 3 = TEXT+UTF+PACK+MM+LZX&HUFFMAN", true); log.println(" 4 = TEXT+UTF+EXE+PACK+MM+ROLZ&NONE", true); log.println(" 5 = TEXT+UTF+BWT+RANK+ZRLT&ANS0", true); log.println(" 6 = TEXT+UTF+BWT+SRT+ZRLT&FPAQ", true); log.println(" 7 = LZP+TEXT+UTF+BWT+LZP&CM", true); log.println(" 8 = EXE+RLT+TEXT+UTF+DNA&TPAQ", true); log.println(" 9 = EXE+RLT+TEXT+UTF+DNA&TPAQX\n", true); log.println(" Kanzi utilizes various algorithm combinations across different compression", true); log.println(" levels. While these levels are calibrated to improve compression monotonically", true); log.println(" this is not guaranteed. Certain data may respond better to specific algorithms", true); log.println(" meaning higher compression levels could occasionally yield lower compression ratios.\n", true); log.println(" -e, --entropy=", true); log.println(" Entropy codec [None|Huffman|ANS0|ANS1|Range|FPAQ|TPAQ|TPAQX|CM]\n", true); log.println(" -t, --transform=", true); log.println(" Transform [None|BWT|BWTS|LZ|LZX|LZP|ROLZ|ROLZX|RLT|ZRLT]", true); log.println(" [MTFT|RANK|SRT|TEXT|MM|EXE|UTF|PACK]", true); log.println(" EG: BWT+RANK or BWTS+MTFT\n", true); log.println(" -x, -x32, -x64, --checksum=", true); log.println(" Enable block checksum (32 or 64 bits).", true); log.println(" -x is equivalent to -x32.\n", true); log.println(" -s, --skip", true); log.println(" Copy blocks with high entropy instead of compressing them.\n", true); } log.println(" -j, --jobs=", true); log.println(" Maximum number of jobs the program may start concurrently", true); #ifdef CONCURRENCY_ENABLED int cores = min(max(int(thread::hardware_concurrency()) / 2, 1), MAX_CONCURRENCY); char msg[96]; snprintf(msg, sizeof(msg), " Default is half of available cores (%d on this machine).\n", cores); log.println(" If 0 is provided, use all available cores (maximum is 64).", true); log.println(msg, true); #else log.println(" (always 1 in this version).\n", true); #endif log.println(" -v, --verbose=", true); log.println(" 0=silent, 1=default, 2=display details, 3=display configuration,", true); log.println(" 4=display block size and timings, 5=display extra information", true); log.println(" Verbosity is reduced to 1 when files are processed concurrently.", true); log.println(" Verbosity is reduced to 0 when the output is 'stdout'.\n", true); if (mode != "y") { log.println(" -f, --force", true); log.println(" Overwrite the output file if it already exists\n", true); } if (mode == "c") { log.println(" --rm", true); log.println(" Remove the input file after successful compression.", true); log.println(" If the input is a folder, all processed files under the folder are removed.\n", true); } else if (mode == "d") { log.println(" --rm", true); log.println(" Remove the input file after successful decompression.", true); log.println(" If the input is a folder, all processed files under the folder are removed.\n", true); } log.println(" --skip-links", true); log.println(" Do not follow links\n", true); log.println(" --skip-dot-files", true); log.println(" Skip dot files\n", true); if (mode == "d") { log.println(" --from=blockId", true); log.println(" Decompress starting at the provided block (included).", true); log.println(" The first block ID is 1.\n", true); log.println(" --to=blockId", true); log.println(" Decompress ending at the provided block (excluded).\n", true); log.println("", true); log.println("Examples\n", true); log.println(" kanzi -d -i foo.knz -f -v 2 -j 2\n", true); log.println(" kanzi --decompress --input=foo.knz --force --verbose=2 --jobs=2\n", true); } if (mode == "c") { log.println("", true); log.println("Transforms\n", true); log.println(" BWT: Burrows Wheeler Transform is a transform that reorders symbols", true); log.println(" in a reversible way that is more amenable to entropy coding.", true); log.println(" This implementation uses a linear time forward transform and parallel", true); log.println(" inverse transform.\n", true); log.println(" BWTS: Burrows Wheeler Transform by Scott is a bijective variant of the BWT.\n", true); log.println(" LZ: Lempel Ziv implementation of the dictionary based LZ77 transform that", true); log.println(" removes redundancy in the data.\n", true); log.println(" LZX: Lempel Ziv Extra. Same as above with a bigger hash table and more", true); log.println(" match searches.\n", true); log.println(" LZP: Lempel Ziv Prediction can be described as an LZ implementation with only", true); log.println(" one possible match (no offset is emitted).\n", true); log.println(" RLT: Run Length Transform is a simple transform that replaces runs of similar", true); log.println(" symbols with a compact representation.\n", true); log.println(" ZRLT: Zero Run Length Transform. Similar to RLT but only processes runs of 0.", true); log.println(" Usually used post BWT.\n", true); log.println(" MTFT: Move-To-Front Transform is a transform that reduces entropy by assigning", true); log.println(" shorter symbols to recent data (like a LRU cache). Usually used post BWT.\n", true); log.println(" RANK: Rank Transform is a transform that that reduces entropy by assigning shorter", true); log.println(" symbols based on symbol frequency ranks. Usually used post BWT.\n", true); log.println(" EXE: a transform that reduces the entropy of executable files (X86 & ARM64)", true); log.println(" by replacing relative jump addresses with absolute ones.\n", true); log.println(" TEXT: a text transform that uses a dictionary to replace common words with", true); log.println(" their dictionary index.\n", true); log.println(" ROLZ: Reduced Offset Lempel Ziv is an implementation of LZ that replaces match offsets", true); log.println(" with indexes, creating a more compact output with slower decoding speeds.\n", true); log.println(" ROLZX: Extended ROLZ with more match searches and a more compact encoding.\n", true); log.println(" SRT: Sorted Rank Transform is a transform that that reduces entropy by assigning", true); log.println(" shorter symbols based on symbol frequency ranks. Usually used post BWT.\n", true); log.println(" MM: Multimedia transform is a fast transform that removes redundancy in correlated", true); log.println(" channels in some multimedia files (EG. wav, pnm).\n", true); log.println(" UTF: a fast transform replacing UTF-8 codewords with aliases based on frequencies.\n", true); log.println(" PACK: a fast transform replacing unused symbols with aliases based on frequencies.\n", true); log.println(" DNA: same as PACK but triggered only when DNA data is detected.\n", true); log.println("", true); log.println("Entropy codecs\n", true); log.println(" Huffman: a fast implementation of canonical Huffman. Both encoder and decoder", true); log.println(" use code tables and multi-streams to improve performance.\n", true); log.println(" RANGE: a fast implementation of a static range codec.\n", true); log.println(" ANS: based on Range Asymmetric Numeral Systems by Jarek Duda (specifically", true); log.println(" an implementation by Fabian Giesen). Works in a similar fashion to the Range", true); log.println(" codec but uses only 1 state instead of 2, and encodes in reverse byte order.\n", true); log.println(" FPAQ: a binary arithmetic codec based on FPAQ1 by Matt Mahoney. Uses a simple", true); log.println(" adaptive order 0 predictor based on frequencies.\n", true); log.println(" CM: a binary arithmetic codec derived from BCM by Ilya Muravyov. Uses context", true); log.println(" mixing of counters to generate a prediction of the next bit value.\n", true); log.println(" TPAQ: a binary arithmetic codec based initially on Tangelo 2.4 (itself derived", true); log.println(" from FPAQ8). Uses context mixing of predictions produced by one layer", true); log.println(" neural networks. The initial code has been heavily tuned to improve", true); log.println(" compression ratio and speed. Slow but usually excellent compression ratio.\n", true); log.println(" TPAQX: Extended TPAQ with more predictions and more memory usage. Slowest but", true); log.println(" usually the best compression ratio.\n", true); log.println("", true); log.println("Examples\n", true); log.println(" kanzi -c -i foo.txt -o none -b 4m -l 4 -v 3\n", true); log.println(" kanzi -c -i foo.txt -f -t BWT+MTFT+ZRLT -b 4m -e FPAQ -j 4\n", true); log.println(" kanzi --compress --input=foo.txt --output=foo.knz --force", true); log.println(" --transform=BWT+MTFT+ZRLT --block=4m --entropy=FPAQ --jobs=4\n", true); } log.println("", true); log.println("Credits\n", true); log.println(" Matt Mahoney, Yann Collet, Jan Ondrus, Yuta Mori, Ilya Muravyov,", true); log.println(" Neal Burns, Fabian Giesen, Jarek Duda, Ilya Grebnov\n", true); } void printHeader(Printer& log, int verbose, bool& showHeader) { if (verbose < 1) return; log.println("", true); log.println(APP_HEADER, true); log.println("", verbose > 1); log.println(APP_SUB_HEADER, verbose > 1); if (verbose >= 5) { stringstream extraHeader; #ifdef __clang__ extraHeader << "Compiled with clang version "; extraHeader << __clang_major__ << "." << __clang_minor__; #else #ifdef _MSC_VER extraHeader << "Compiled with Visual Studio"; #ifdef _MSC_VER_STR // see types.h extraHeader << " " << _MSC_VER_STR; #endif #else #ifdef __INTEL_COMPILER extraHeader << "Compiled with Intel compiler "; extraHeader << "(" << __INTEL_COMPILER_BUILD_DATE << ")"; #else #ifdef __GNUC__ extraHeader << "Compiled with gcc version "; extraHeader << __GNUC__ << "." << __GNUC_MINOR__ << "." << __GNUC_PATCHLEVEL__; #endif #endif #endif #endif if (extraHeader.str().length() > 0) { #if defined(__AVX2__) extraHeader << " - AVX2"; #elif defined(__AVX__) extraHeader << " - AVX"; #elif defined(__AVX512F__) extraHeader << " - AVX512"; #elif defined(__SSE4_1__) extraHeader << " - SSE4.1"; #elif defined(__SSE3__) extraHeader << " - SSE3"; #elif defined(__SSE2__) extraHeader << " - SSE2"; #elif defined(__SSE__) extraHeader << " - SSE"; #endif log.println(extraHeader.str(), true); } } showHeader = false; } #define WARNING_OPT_NOVALUE(opt) \ stringstream ss; \ ss << "Warning: ignoring option [" << opt << "] with no value."; \ log.println(ss.str(), verbose > 0) #define WARNING_OPT_COMP_ONLY(opt) \ stringstream ss; \ ss << "Warning: ignoring option [" << opt << "]. Only applicable in compression mode."; \ log.println(ss.str(), verbose > 0) #define WARNING_OPT_DECOMP_ONLY(opt) \ stringstream ss; \ ss << "Warning: ignoring option [" << opt << "]. Only applicable in decompression mode."; \ log.println(ss.str(), verbose > 0) #define WARNING_OPT_INVALID(opt) \ stringstream ss; \ ss << "Warning: ignoring option [" << opt << "]. Not applicable in this mode."; \ log.println(ss.str(), verbose > 0) #define WARNING_OPT_DUPLICATE(opt, val) \ stringstream ss; \ ss << "Warning: ignoring duplicate option [" << opt << "]: " << val;\ log.println(ss.str(), verbose > 0) static bool toInt(const string& s, int& res) { if (s.empty()) return false; int value = 0; for (size_t i = 0; i < s.length(); i++) { if ((s[i] < '0') || (s[i] > '9')) return false; const int digit = s[i] - '0'; if (value > ((numeric_limits::max)() - digit) / 10) return false; value = value * 10 + digit; } res = value; return true; } int processCommandLine(int argc, const char* argv[], Context& map, Printer& log) { string inputName; string outputName; int remove = -1; int overwrite = -1; int checksum = 0; int skip = -1; int reorder = -1; int noDotFiles = -1; int noLinks = -1; string codec; string transf; bool verboseFlag = false; int verbose = 1; int ctx = -1; int level = -1; int from = -1; int to = -1; int tasks = -1; int blockSize = -1; int autoBlockSize = -1; string mode; bool showHeader = true; bool showHelp = false; for (int i = 1; i < argc; i++) { string arg(argv[i]); trim(arg); if (arg == "-v") { ctx = ARG_IDX_VERBOSE; continue; } if (arg == "-i") { ctx = ARG_IDX_INPUT; continue; } if (arg == "-o") { ctx = ARG_IDX_OUTPUT; continue; } // Extract verbosity, output and mode first if ((arg == "-c") || (arg.compare(0, 10, "--compress") == 0)) { if (mode != "") { cerr << "Only one mode can be provided (already got '" << mode << "'" << endl; return Error::ERR_INVALID_PARAM; } mode = "c"; continue; } if ((arg == "-d") || (arg.compare(0, 12, "--decompress") == 0)) { if (mode != "") { cerr << "Only one mode can be provided (already got '" << mode << "'" << endl; return Error::ERR_INVALID_PARAM; } mode = "d"; continue; } if ((arg == "-y") || (arg.compare(0, 10, "--info") == 0)) { if (mode != "") { cerr << "Only one mode can be provided (already got '" << mode << "')" << endl; return Error::ERR_INVALID_PARAM; } mode = "y"; continue; } if ((ctx == ARG_IDX_VERBOSE) || (arg.compare(0, 10, "--verbose=") == 0)) { if (verboseFlag == true) { WARNING_OPT_DUPLICATE("verbosity level", arg); } else { if (ctx != ARG_IDX_VERBOSE) arg = arg.substr(10); if ((toInt(arg, verbose) == false) || (verbose < 0) || (verbose > 5)) { cerr << "Invalid verbosity level provided on command line: " << arg << endl; return Error::ERR_INVALID_PARAM; } verboseFlag = true; } } else if ((ctx == ARG_IDX_OUTPUT) || (arg.compare(0, 9, "--output=") == 0)) { if (ctx != ARG_IDX_OUTPUT) arg = arg.substr(9); outputName = trim(arg); } else if ((ctx == ARG_IDX_INPUT) || (arg.compare(0, 8, "--input=") == 0)) { if (ctx != ARG_IDX_INPUT) arg = arg.substr(8); inputName = trim(arg); } else if ((arg == "--help") || (arg == "-h")) { showHelp = true; } ctx = -1; } if ((argc == 1) || (showHelp == true)) { printHeader(log, verbose, showHeader); printHelp(log, mode, showHeader); return 0; } // Overwrite verbosity if the output goes to stdout if (outputName.length() == 0) { if (inputName.length() == 0) { verbose = 0; verboseFlag = true; } } else { if (mode == "y") { WARNING_OPT_INVALID(outputName); } else { string str = outputName; transform(str.begin(), str.end(), str.begin(), safeToUpper); if (str == "STDOUT") { verbose = 0; verboseFlag = true; } } } printHeader(log, verbose, showHeader); inputName.clear(); outputName.clear(); ctx = -1; for (int i = 1; i < argc; i++) { string arg(argv[i]); if ((arg.length() > 0) && (arg[0] == 0x20)) { size_t k = 1; // Left trim limited to spaces (due to possible unicode chars in names) while ((k < arg.length()) && (arg[k] == 0x20)) k++; arg = arg.substr(k); } if ((arg == "-c") || (arg == "-d") || (arg == "-y") || (arg == "--compress") || (arg == "--decompress") || (arg == "--info")) { if (ctx != -1) { WARNING_OPT_NOVALUE(CMD_LINE_ARGS[ctx]); } ctx = -1; continue; } if ((arg == "--force") || (arg == "-f")) { if (ctx != -1) { WARNING_OPT_NOVALUE(CMD_LINE_ARGS[ctx]); } else if (overwrite >= 0) { WARNING_OPT_DUPLICATE(arg, "true"); } if (mode == "y") { WARNING_OPT_INVALID(arg); ctx = - 1; continue; } overwrite = 1; ctx = -1; continue; } if ((arg == "--skip") || (arg == "-s")) { if (ctx != -1) { WARNING_OPT_NOVALUE(CMD_LINE_ARGS[ctx]); } else if (skip >= 0) { WARNING_OPT_DUPLICATE(arg, "true"); } ctx = -1; if (mode != "c") { WARNING_OPT_COMP_ONLY(arg); continue; } skip = 1; continue; } if ((arg == "-x") || (arg == "-x32") || (arg == "-x64")) { if (checksum > 0) { WARNING_OPT_DUPLICATE(arg, "true"); } ctx = -1; if (mode != "c") { WARNING_OPT_COMP_ONLY(arg); continue; } checksum = (arg == "-x64") ? 64 : 32; continue; } if (arg == "--rm") { if (ctx != -1) { WARNING_OPT_NOVALUE(CMD_LINE_ARGS[ctx]); } else if (remove >= 0) { WARNING_OPT_DUPLICATE(arg, "true"); } ctx = -1; if (mode == "y") { WARNING_OPT_INVALID(arg); continue; } remove = 1; continue; } if (arg == "--no-file-reorder") { if (ctx != -1) { WARNING_OPT_NOVALUE(CMD_LINE_ARGS[ctx]); } else if (reorder >= 0) { WARNING_OPT_DUPLICATE(arg, "true"); } ctx = -1; if (mode != "c") { WARNING_OPT_COMP_ONLY(arg); continue; } reorder = 0; continue; } if (arg == "--skip-dot-files") { if (ctx != -1) { WARNING_OPT_NOVALUE(CMD_LINE_ARGS[ctx]); } else if (noDotFiles >= 0) { WARNING_OPT_DUPLICATE(arg, "true"); } ctx = -1; noDotFiles = 1; continue; } if (arg == "--skip-links") { if (ctx != -1) { WARNING_OPT_NOVALUE(CMD_LINE_ARGS[ctx]); } else if (noLinks >= 0) { WARNING_OPT_DUPLICATE(arg, "true"); } ctx = -1; noLinks = 1; continue; } if (ctx == -1) { for (int j = 0; j < 10; j++) { if (arg == CMD_LINE_ARGS[j]) { ctx = j; break; } } if (ctx != -1) continue; } if ((ctx == ARG_IDX_OUTPUT) || (arg.compare(0, 9, "--output=") == 0)) { if (ctx != ARG_IDX_OUTPUT) arg = arg.substr(9); if (outputName != "") { string msg = (ctx == ARG_IDX_OUTPUT) ? CMD_LINE_ARGS[ctx] : arg; if (mode == "y") { WARNING_OPT_INVALID(msg); ctx = -1; continue; } WARNING_OPT_DUPLICATE(msg, arg); } else { if (mode == "y") { string msg = (ctx == ARG_IDX_OUTPUT) ? CMD_LINE_ARGS[ctx] : arg; WARNING_OPT_INVALID(msg); ctx = -1; continue; } if ((arg.length() >= 2) && (arg[0] == '.') && (arg[1] == PATH_SEPARATOR)) { arg = (arg.length() == 2) ? arg.substr(0, 1) : arg.substr(2); } outputName = arg; } ctx = -1; continue; } if ((ctx == ARG_IDX_INPUT) || (arg.compare(0, 8, "--input=") == 0)) { if (ctx != ARG_IDX_INPUT) arg = arg.substr(8); if (inputName != "") { string msg = (ctx == ARG_IDX_INPUT) ? CMD_LINE_ARGS[ctx] : arg; WARNING_OPT_DUPLICATE(msg, arg); } else { if ((arg.length() >= 2) && (arg[0] == '.') && (arg[1] == PATH_SEPARATOR)) { arg = (arg.length() == 2) ? arg.substr(0, 1) : arg.substr(2); } inputName = arg; } ctx = -1; continue; } if ((ctx == ARG_IDX_ENTROPY) || (arg.compare(0, 10, "--entropy=") == 0)) { if (ctx != ARG_IDX_ENTROPY) arg = arg.substr(10); if (mode != "c"){ string msg = (ctx == ARG_IDX_ENTROPY) ? CMD_LINE_ARGS[ctx] : arg; WARNING_OPT_COMP_ONLY(msg); ctx = -1; continue; } if (codec != "") { string msg = (ctx == ARG_IDX_ENTROPY) ? CMD_LINE_ARGS[ctx] : arg; WARNING_OPT_DUPLICATE(msg, arg); } else { if (arg.length() == 0) { cerr << "Invalid empty entropy provided on command line" << endl; return Error::ERR_INVALID_PARAM; } codec = arg; transform(codec.begin(), codec.end(), codec.begin(), safeToUpper); } ctx = -1; continue; } if ((ctx == ARG_IDX_TRANSFORM) || (arg.compare(0, 12, "--transform=") == 0)) { if (ctx != ARG_IDX_TRANSFORM) arg = arg.substr(12); if (mode != "c"){ string msg = (ctx == ARG_IDX_TRANSFORM) ? CMD_LINE_ARGS[ctx] : arg; WARNING_OPT_COMP_ONLY(msg); ctx = -1; continue; } if (transf != "") { string msg = (ctx == ARG_IDX_TRANSFORM) ? CMD_LINE_ARGS[ctx] : arg; WARNING_OPT_DUPLICATE(msg, arg); } else { if (arg.length() == 0) { cerr << "Invalid empty transform provided on command line" << endl; return Error::ERR_INVALID_PARAM; } transf = arg; transform(transf.begin(), transf.end(), transf.begin(), safeToUpper); } while ((transf.length() > 0) && (transf[0] == '+')) { transf = transf.substr(1); } while ((transf.length() > 0) && (transf[transf.length() - 1] == '+')) { transf.resize(transf.length() - 1); } ctx = -1; continue; } if ((ctx == ARG_IDX_LEVEL) || (arg.compare(0, 8, "--level=") == 0)) { if (ctx != ARG_IDX_LEVEL) arg = arg.substr(8); if (mode != "c") { string msg = (ctx == ARG_IDX_LEVEL) ? CMD_LINE_ARGS[ctx] : arg; WARNING_OPT_COMP_ONLY(msg); ctx = -1; continue; } if (level >= 0) { string msg = (ctx == ARG_IDX_LEVEL) ? CMD_LINE_ARGS[ctx] : arg; WARNING_OPT_DUPLICATE(msg, arg); } else { if ((toInt(arg, level) == false) || ((level < 0) || (level > 9))) { cerr << "Invalid compression level provided on command line: " << arg << endl; return Error::ERR_INVALID_PARAM; } } ctx = -1; continue; } if (arg.compare(0, 11, "--checksum=") == 0) { arg = arg.substr(11); if (mode != "c") { WARNING_OPT_COMP_ONLY(arg); ctx = -1; continue; } if (checksum > 0) { WARNING_OPT_DUPLICATE("--checksum", arg); } else { if ((toInt(arg, checksum) == false) || ((checksum != 32) && (checksum != 64))) { cerr << "Invalid block checksum size provided on command line: " << arg << endl; return Error::ERR_INVALID_PARAM; } } ctx = -1; continue; } if ((ctx == ARG_IDX_BLOCK) || (arg.compare(0, 8, "--block=") == 0)) { if (ctx != ARG_IDX_BLOCK) arg = arg.substr(8); if (arg.length() == 0) { cerr << "Invalid block size provided on command line: " << arg << endl; return Error::ERR_INVALID_PARAM; } if (mode != "c") { string msg = (ctx == ARG_IDX_BLOCK) ? CMD_LINE_ARGS[ctx] : arg; WARNING_OPT_COMP_ONLY(msg); ctx = -1; continue; } if ((blockSize >= 0) || (autoBlockSize >= 0)) { string msg = (ctx == ARG_IDX_BLOCK) ? CMD_LINE_ARGS[ctx] : arg; WARNING_OPT_DUPLICATE(msg, arg); ctx = -1; continue; } transform(arg.begin(), arg.end(), arg.begin(), safeToUpper); if (arg == "AUTO") { autoBlockSize = 1; } else { uint64 scale = 1; char lastChar = arg[arg.length() - 1]; // Process K or M or G suffix if ('K' == lastChar) { scale = 1024; arg.resize(arg.length() - 1); } else if ('M' == lastChar) { scale = 1024 * 1024; arg.resize(arg.length() - 1); } else if ('G' == lastChar) { scale = 1024 * 1024 * 1024; arg.resize(arg.length() - 1); } if ((toInt(arg, blockSize) == false) || (uint64(blockSize) > uint64((numeric_limits::max)()) / scale)) { cerr << "Invalid block size provided on command line: " << arg << endl; return Error::ERR_INVALID_PARAM; } blockSize = int(uint64(blockSize) * scale); } ctx = -1; continue; } if ((ctx == ARG_IDX_JOBS) || (arg.compare(0, 7, "--jobs=") == 0)) { if (ctx != ARG_IDX_JOBS) arg = arg.substr(7); if (tasks >= 0) { string msg = (ctx == ARG_IDX_BLOCK) ? CMD_LINE_ARGS[ctx] : arg; WARNING_OPT_DUPLICATE(msg, arg); ctx = -1; continue; } if ((toInt(arg, tasks) == false) || (tasks < 0)) { cerr << "Invalid number of jobs provided on command line: " << arg << endl; return Error::ERR_INVALID_PARAM; } ctx = -1; continue; } if ((arg.compare(0, 7, "--from=") == 0) && (ctx == -1)) { if (mode != "d"){ WARNING_OPT_DECOMP_ONLY("--from"); continue; } arg = arg.substr(7); if (from >= 0) { WARNING_OPT_DUPLICATE("--from", arg); } else { if ((toInt(arg, from) == false) || (from <= 0)) { cerr << "Invalid start block provided on command line: " << arg << endl; if (from == 0) { cerr << "The first block ID is 1." << endl; } return Error::ERR_INVALID_PARAM; } } continue; } if ((arg.compare(0, 5, "--to=") == 0) && (ctx == -1)) { if (mode != "d"){ WARNING_OPT_DECOMP_ONLY("--to"); continue; } arg = arg.substr(5); if (to >= 0) { WARNING_OPT_DUPLICATE("--to", arg); } else { if ((toInt(arg, to) == false) || (to <= 0)) { // Must be > 0 (0 means nothing to do) cerr << "Invalid end block provided on command line: " << arg << endl; return Error::ERR_INVALID_PARAM; } } continue; } if ((arg.compare(0, 10, "--verbose=") != 0) && (ctx == -1)) { stringstream ss; ss << "Warning: ignoring unknown option [" << arg << "]"; log.println(ss.str(), verbose > 0); } ctx = -1; } if (ctx != -1) { stringstream ss; ss << "Warning: ignoring option with missing value [" << CMD_LINE_ARGS[ctx] << "]"; log.println(ss.str(), verbose > 0); } if (level >= 0) { if (codec.length() > 0) { stringstream ss; ss << "Warning: providing the 'level' option forces the entropy codec. Ignoring [" << codec << "]"; log.println(ss.str(), verbose > 0); } if (transf.length() > 0) { stringstream ss; ss << "Warning: providing the 'level' option forces the transform. Ignoring [" << transf << "]"; log.println(ss.str(), verbose > 0); } } if (blockSize >= 0) map.putInt("blockSize", blockSize); map.putInt("verbosity", (verboseFlag == false) ? 1 : verbose); map.putString("mode", mode); map.putString("inputName", inputName); map.putString("outputName", outputName); map.putInt("checksum", checksum); if (autoBlockSize == 1) map.putInt("autoBlock", 1); if ((mode == "c") && (level >= 0)) map.putInt("level", level); if (overwrite == 1) map.putInt("overwrite", 1); if (remove == 1) map.putInt("remove", 1); if (codec.length() > 0) map.putString("entropy", codec); if (transf.length() > 0) map.putString("transform", transf); if (skip == 1) map.putInt("skipBlocks", 1); if (reorder == 0) map.putInt("fileReorder", 0); else map.putInt("fileReorder", 1); if (noDotFiles == 1) // Skip dot files map.putInt("noDotFiles", 1); if (noLinks == 1) // Do not follow links map.putInt("noLinks", 1); if (from >= 0) map.putInt("from", from); if (to >= 0) map.putInt("to", to); if (tasks >= 0) map.putInt("jobs", tasks); return 0; } int main(int argc, const char* argv[]) { #if defined(WIN32) || defined(_WIN32) || defined(_WIN64) // Force standard input and output to binary mode (void) _setmode(_fileno(stdout), _O_BINARY); (void) _setmode(_fileno(stdin), _O_BINARY); // Users can provide a custom code page to properly display some non ASCII file names // eg. 1252 for ANSI Latin-1 or 65001 for utf-8 size_t size; getenv_s(&size, nullptr, 0, "KANZI_CODE_PAGE"); if (size != 0) { char* p = new char[size]; getenv_s(&size, p, size, "KANZI_CODE_PAGE"); string s(p); int cp; if (toInt(s, cp) == true) { SetConsoleCP(cp); SetConsoleOutputCP(cp); } delete[] p; } #endif Context ctx; Printer log(cout); try { int status = processCommandLine(argc, argv, ctx, log); // Command line processing error ? if (status != 0) return status; // Help mode only ? if (ctx.has("mode") == false) return 0; string mode = ctx.getString("mode"); int jobs = ctx.getInt("jobs", -1); #ifndef CONCURRENCY_ENABLED if (jobs > 1) { const int verbosity = ctx.getInt("verbosity"); stringstream ss; ss << "Warning: the number of jobs is limited to 1 in this version"; log.println(ss.str(), verbosity > 0); } jobs = 1; #else if (jobs == 0) { int cores = max(int(thread::hardware_concurrency()), 1); // User provided 0 => use all the cores jobs = min(cores, MAX_CONCURRENCY); } else if (jobs == -1) { int cores = max(int(thread::hardware_concurrency()) / 2, 1); // Defaults to half the cores jobs = min(cores, MAX_CONCURRENCY); } else if (jobs > MAX_CONCURRENCY) { const int verbosity = ctx.getInt("verbosity"); stringstream ss; ss << "Warning: the number of jobs is too high, defaulting to " << MAX_CONCURRENCY; log.println(ss.str(), verbosity > 0); jobs = MAX_CONCURRENCY; } #endif ctx.putInt("jobs", jobs); if (mode == "c") { try { BlockCompressor bc(ctx); uint64 written = 0; int code = bc.compress(written); return code; } catch (const exception& e) { cerr << "Could not create the compressor: " << e.what() << endl; return Error::ERR_CREATE_COMPRESSOR; } } if ((mode == "d") || (mode == "y")) { try { BlockDecompressor bd(ctx); uint64 read = 0; int code = bd.decompress(read); return code; } catch (const exception& e) { cerr << "Could not create the decompressor: " << e.what() << endl; return Error::ERR_CREATE_DECOMPRESSOR; } } cout << "Missing arguments: try --help or -h" << endl; return Error::ERR_MISSING_PARAM; } #if __cplusplus >= 201703L catch (const bad_variant_access& e) { // May be thrown by Context cerr << e.what() << endl; return Error::ERR_UNKNOWN; } #endif catch (const invalid_argument& e) { // May be thrown by ThreadPool cerr << e.what() << endl; return Error::ERR_INVALID_PARAM; } catch (const runtime_error& e) { // May be thrown by ThreadPool cerr << e.what() << endl; return Error::ERR_INVALID_PARAM; } } kanzi-cpp-2.6.0/src/bitstream/000077500000000000000000000000001525576310500162055ustar00rootroot00000000000000kanzi-cpp-2.6.0/src/bitstream/DebugInputBitStream.cpp000066400000000000000000000076531525576310500226050ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include "../bitstream/DebugInputBitStream.hpp" using namespace kanzi; using namespace std; DebugInputBitStream::DebugInputBitStream(InputBitStream& ibs) : _delegate(ibs), _out(cout), _width(80) { _idx = 0; _mark = false; _show = false; _hexa = false; _current = kanzi::byte(0); } DebugInputBitStream::DebugInputBitStream(InputBitStream& ibs, ostream& os) : _delegate(ibs), _out(os), _width(80) { _idx = 0; _mark = false; _show = false; _hexa = false; _current = kanzi::byte(0); } DebugInputBitStream::DebugInputBitStream(InputBitStream& ibs, ostream& os, int width) : _delegate(ibs), _out(os) { if ((width != -1) && (width < 8)) width = 8; if (width != -1) width &= 0xFFFFFFF8; _width = width; _idx = 0; _mark = false; _show = false; _hexa = false; _current = kanzi::byte(0); } DebugInputBitStream::~DebugInputBitStream() { _close(); } // Returns 1 or 0 int DebugInputBitStream::readBit() { int res = _delegate.readBit(); _current <<= 1; _current |= kanzi::byte(res); _out << ((res & 1) == 1 ? "1" : "0"); _idx++; if (_mark == true) _out << "r"; if ((_width != -1) && ((_idx - 1) % _width == _width - 1)) { if (showByte()) printByte(_current); _out << endl; _idx = 0; } else if ((_idx & 7) == 0) { if (showByte()) printByte(_current); else _out << " "; } return res; } uint64 DebugInputBitStream::readBits(uint count) { uint64 res = _delegate.readBits(count); for (uint i = 1; i <= count; i++) { int bit = (res >> (count - i)) & 1; _idx++; _current <<= 1; _current |= kanzi::byte(bit); _out << ((bit == 1) ? "1" : "0"); if ((_mark == true) && (i == count)) _out << "r"; if ((_width != -1) && (_idx % _width == 0)) { if (showByte()) printByte(_current); _out << endl; _idx = 0; } else if ((_idx & 7) == 0) { if (showByte()) printByte(_current); else _out << " "; } } return res; } uint DebugInputBitStream::readBits(kanzi::byte bits[], uint count) { count = _delegate.readBits(bits, count); for (uint i = 0; i < count; i++) { const int bit = int(bits[i >> 3] >> (7 - (i & 7))) & 1; _idx++; _current <<= 1; _current |= kanzi::byte(bit); _out << ((bit == 1) ? "1" : "0"); if ((_mark == true) && (i + 1 == count)) _out << "r"; if ((_width != -1) && (_idx % _width == 0)) { if (showByte()) printByte(_current); _out << endl; _idx = 0; } else if ((_idx & 7) == 0) { if (showByte()) printByte(_current); else _out << " "; } } return count; } void DebugInputBitStream::printByte(kanzi::byte b) { int val = int(b); if (_hexa == true) { _out << hex << " [0x"; _out << ((val < 16) ? "0" : ""); _out << val << "] "; _out << dec; return; } _out << " ["; if (val < 10) _out << "00"; else if (val < 100) _out << "0"; _out << val << "] "; } kanzi-cpp-2.6.0/src/bitstream/DebugInputBitStream.hpp000066400000000000000000000037571525576310500226130ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_DebugInputBitStream #define knz_DebugInputBitStream #include "../InputBitStream.hpp" #include "../OutputStream.hpp" namespace kanzi { class DebugInputBitStream FINAL : public InputBitStream { private: InputBitStream& _delegate; OutputStream& _out; int _width; int _idx; bool _mark; bool _hexa; bool _show; byte _current; void printByte(byte val); void _close() { _delegate.close(); } public: DebugInputBitStream(InputBitStream& ibs); DebugInputBitStream(InputBitStream& ibs, OutputStream& os); DebugInputBitStream(InputBitStream& ibs, OutputStream& os, int width); ~DebugInputBitStream(); // Returns 1 or 0 int readBit(); uint64 readBits(uint length); uint readBits(byte bits[], uint length); // Number of bits read uint64 read() const { return _delegate.read(); } // Return false when the bitstream is closed or the End-Of-Stream has been reached bool hasMoreToRead() { return _delegate.hasMoreToRead(); } void close() { _close(); } void showByte(bool show) { _show = show; } void setHexa(bool hexa) { _hexa = hexa; } bool hexa() const { return _hexa; } bool showByte() const { return _show; } void setMark(bool mark) { _mark = mark; } bool mark() const { return _mark; } }; } #endif kanzi-cpp-2.6.0/src/bitstream/DebugOutputBitStream.cpp000066400000000000000000000076341525576310500230050ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include "DebugOutputBitStream.hpp" using namespace kanzi; using namespace std; DebugOutputBitStream::DebugOutputBitStream(OutputBitStream& obs) : _delegate(obs), _out(cout), _width(80) { _mark = false; _hexa = false; _show = false; _current = kanzi::byte(0); _idx = 0; } DebugOutputBitStream::DebugOutputBitStream(OutputBitStream& obs, OutputStream& os) : _delegate(obs), _out(os), _width(80) { _mark = false; _hexa = false; _show = false; _current = kanzi::byte(0); _idx = 0; } DebugOutputBitStream::DebugOutputBitStream(OutputBitStream& obs, OutputStream& os, int width) : _delegate(obs), _out(os) { if ((width != -1) && (width < 8)) width = 8; if (width != -1) width &= 0xFFFFFFF8; _width = width; _mark = false; _hexa = false; _show = false; _current = kanzi::byte(0); _idx = 0; } DebugOutputBitStream::~DebugOutputBitStream() { _close(); } void DebugOutputBitStream::writeBit(int bit) { bit &= 1; _out << ((bit == 1) ? "1" : "0"); _current <<= 1; _current |= kanzi::byte(bit); _idx++; if (_mark == true) _out << "w"; if ((_width != -1) && ((_idx - 1) % _width == _width - 1)) { if (showByte()) printByte(_current); _out << endl; _idx = 0; } else if ((_idx & 7) == 0) { if (showByte()) printByte(_current); else _out << " "; } _delegate.writeBit(bit); } uint DebugOutputBitStream::writeBits(uint64 bits, uint count) { uint res = _delegate.writeBits(bits, count); for (uint i = 1; i <= res; i++) { uint64 bit = (bits >> (res - i)) & 1; _current <<= 1; _current |= kanzi::byte(bit); _idx++; _out << ((bit == 1) ? "1" : "0"); if ((_mark == true) && (i == res)) _out << "w"; if ((_width != -1) && (_idx % _width == 0)) { if (showByte()) printByte(_current); _out << endl; _idx = 0; } else if ((_idx & 7) == 0) { if (showByte()) printByte(_current); else _out << " "; } } return res; } uint DebugOutputBitStream::writeBits(const kanzi::byte bits[], uint count) { uint res = _delegate.writeBits(bits, count); for (uint i = 0; i < res; i++) { const uint64 bit = uint64(bits[i >> 3] >> (7 - (i & 7))) & 1; _current <<= 1; _current |= kanzi::byte(bit); _idx++; _out << ((bit == 1) ? "1" : "0"); if ((_mark == true) && (i + 1 == res)) _out << "w"; if ((_width != -1) && (_idx % _width == 0)) { if (showByte()) printByte(_current); _out << endl; _idx = 0; } else if ((_idx & 7) == 0) { if (showByte()) printByte(_current); else _out << " "; } } return res; } void DebugOutputBitStream::printByte(kanzi::byte b) { int val = int(b); if (_hexa == true) { _out << hex << " [0x"; _out << ((val < 16) ? "0" : ""); _out << val << "] "; _out << dec; return; } _out << " ["; if (val < 10) _out << "00"; else if (val < 100) _out << "0"; _out << val << "] "; } kanzi-cpp-2.6.0/src/bitstream/DebugOutputBitStream.hpp000066400000000000000000000035711525576310500230060ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_DebugOutputBitStream #define knz_DebugOutputBitStream #include "../OutputBitStream.hpp" #include "../OutputStream.hpp" namespace kanzi { class DebugOutputBitStream FINAL : public OutputBitStream { private: OutputBitStream& _delegate; OutputStream& _out; int _width; int _idx; bool _mark; bool _show; bool _hexa; byte _current; void printByte(byte val); void _close() { _delegate.close(); } public: DebugOutputBitStream(OutputBitStream& obs); DebugOutputBitStream(OutputBitStream& obs, OutputStream& os); DebugOutputBitStream(OutputBitStream& obs, OutputStream& os, int width); ~DebugOutputBitStream(); void writeBit(int bit); uint writeBits(uint64 bits, uint length); uint writeBits(const byte bits[], uint length); // Return number of bits written so far uint64 written() const { return _delegate.written(); } void close() { _close(); } void showByte(bool show) { _show = show; } void setHexa(bool hexa) { _hexa = hexa; } bool hexa() const { return _hexa; } bool showByte() const { return _show; } void setMark(bool mark) { _mark = mark; } bool mark() const { return _mark; } }; } #endif kanzi-cpp-2.6.0/src/bitstream/DefaultInputBitStream.cpp000066400000000000000000000157571525576310500231470ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include "DefaultInputBitStream.hpp" using namespace kanzi; using namespace std; DefaultInputBitStream::DefaultInputBitStream(InputStream& is, uint bufferSize) : _is(is) { if (bufferSize < 1024) throw invalid_argument("Invalid buffer size (must be at least 1024)"); if (bufferSize > 1 << 29) throw invalid_argument("Invalid buffer size (must be at most 536870912)"); if ((bufferSize & 7) != 0) throw invalid_argument("Invalid buffer size (must be a multiple of 8)"); _bufferSize = bufferSize; _buffer = new kanzi::byte[_bufferSize]; _availBits = 0; _maxPosition = -1; _position = 0; _current = 0; _read = 0; _closed = false; } DefaultInputBitStream::~DefaultInputBitStream() { _close(); delete[] _buffer; } uint DefaultInputBitStream::readBits(kanzi::byte bits[], uint count) { if (isClosed() == true) throw BitStreamException("Stream closed", BitStreamException::STREAM_CLOSED); if (count == 0) return 0; uint remaining = count; uint start = 0; // Byte aligned cursor ? if ((_availBits & 7) == 0) { if (_availBits == 0) _availBits = pullCurrent(); // Empty _current while ((_availBits > 0) && (remaining >= 8)) { bits[start] = kanzi::byte(readBits(8)); start++; remaining -= 8; } prefetchRead(&_buffer[_position]); uint availBytes = uint(_maxPosition + 1 - _position); // Copy internal buffer to bits array while ((remaining >> 3) > availBytes) { memcpy(&bits[start], &_buffer[_position], availBytes); start += availBytes; remaining -= (availBytes << 3); _position = _maxPosition + 1; const int read = readFromInputStream(_bufferSize); availBytes = uint(_maxPosition + 1 - _position); if (read < int(_bufferSize)) break; } const uint r = min((remaining >> 6) << 3, availBytes); if (r > 0) { memcpy(&bits[start], &_buffer[_position], r); _position += r; start += r; remaining -= (r << 3); } } else if (remaining >= 64) { // Not kanzi::byte aligned const uint a = _availBits; const uint r = 64 - a; while (remaining >= 256) { const uint64 v0 = _current; if (_position + 32 > _maxPosition) { _availBits = pullCurrent(); if (_availBits < r) throw BitStreamException("No more data to read in the bitstream", BitStreamException::END_OF_STREAM); _availBits -= r; BigEndian::writeLong64(&bits[start], (v0 << r) | (_current >> _availBits)); start += 8; remaining -= 64; continue; } const uint64 v1 = BigEndian::readLong64(&_buffer[_position + 0]); const uint64 v2 = BigEndian::readLong64(&_buffer[_position + 8]); const uint64 v3 = BigEndian::readLong64(&_buffer[_position + 16]); const uint64 v4 = BigEndian::readLong64(&_buffer[_position + 24]); _current = v4; _position += 32; BigEndian::writeLong64(&bits[start + 0], (v0 << r) | (v1 >> a)); BigEndian::writeLong64(&bits[start + 8], (v1 << r) | (v2 >> a)); BigEndian::writeLong64(&bits[start + 16], (v2 << r) | (v3 >> a)); BigEndian::writeLong64(&bits[start + 24], (v3 << r) | (v4 >> a)); start += 32; remaining -= 256; } while (remaining >= 64) { const uint64 v = _current; _availBits = pullCurrent(); if (_availBits < r) throw BitStreamException("No more data to read in the bitstream", BitStreamException::END_OF_STREAM); _availBits -= r; BigEndian::writeLong64(&bits[start], (v << r) | (_current >> _availBits)); start += 8; remaining -= 64; } } // Last bytes while (remaining >= 8) { bits[start] = kanzi::byte(readBits(8)); start++; remaining -= 8; } if (remaining > 0) bits[start] = kanzi::byte(readBits(remaining) << (8 - remaining)); return count; } void DefaultInputBitStream::_close() { if (isClosed() == true) return; _closed = true; // Reset fields to force a readFromInputStream() and trigger an exception // on readBit() or readBits() _read -= int64(_availBits); // can be negative _availBits = 0; _maxPosition = -1; } int DefaultInputBitStream::readFromInputStream(uint count) { if (isClosed() == true) throw BitStreamException("Stream closed", BitStreamException::STREAM_CLOSED); if (count == 0) return 0; int size = -1; bool isEOF = false; bool hasReadError = false; try { _read += (int64(_position) << 3); _is.read(reinterpret_cast(_buffer), count); _position = 0; size = (_is.good() == true) ? int(count) : int(_is.gcount()); _maxPosition = (size <= 0) ? -1 : size - 1; isEOF = (_is.eof() == true); hasReadError = (_is.bad() == true) || ((_is.fail() == true) && (isEOF == false)); // Clear flags (required for future seeks when EOF is reached) _is.clear(); } catch (const runtime_error& e) { // Catch IOException without depending on io package throw BitStreamException(e.what(), BitStreamException::INPUT_OUTPUT); } if (hasReadError == true) throw BitStreamException("Read from bitstream failed", BitStreamException::INPUT_OUTPUT); if (size <= 0) { throw BitStreamException((isEOF == true) ? "No more data to read in the bitstream" : "Read from bitstream failed", (isEOF == true) ? BitStreamException::END_OF_STREAM : BitStreamException::INPUT_OUTPUT); } return size; } // Return false when the bitstream is closed or the End-Of-Stream has been reached bool DefaultInputBitStream::hasMoreToRead() { if (isClosed() == true) return false; if ((_position <= _maxPosition) || (_availBits > 0)) return true; try { readFromInputStream(_bufferSize); } catch (const BitStreamException& e) { if (e.error() != BitStreamException::END_OF_STREAM) throw; return false; } return true; } kanzi-cpp-2.6.0/src/bitstream/DefaultInputBitStream.hpp000066400000000000000000000120351525576310500231360ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_DefaultInputBitStream #define knz_DefaultInputBitStream #include "../BitStreamException.hpp" #include "../InputBitStream.hpp" #include "../InputStream.hpp" #include "../Memory.hpp" #include "../Seekable.hpp" #include "../util/strings.hpp" namespace kanzi { #if defined(_MSC_VER) && _MSC_VER <= 1500 class DefaultInputBitStream FINAL : public InputBitStream #else class DefaultInputBitStream FINAL : public InputBitStream, public Seekable #endif { private: InputStream& _is; byte* _buffer; int _position; // index of current byte (consumed if bitIndex == -1) uint _availBits; // bits not consumed in _current int64 _read; uint64 _current; bool _closed; int _maxPosition; uint _bufferSize; int readFromInputStream(uint count); // return number of available bits uint pullCurrent(); void _close(); public: // Returns 1 or 0 int readBit(); uint64 readBits(uint length); uint readBits(byte bits[], uint count); void close() { _close(); } // Number of bits read uint64 read() const { return uint64(_read + (int64(_position) << 3) - int64(_availBits)); } // Return false when the bitstream is closed or the End-Of-Stream has been reached bool hasMoreToRead(); bool isClosed() const { return _closed; } #if !defined(_MSC_VER) || _MSC_VER > 1500 int64 tell(); bool seek(int64 pos); #endif DefaultInputBitStream(InputStream& is, uint bufferSize = 65536); ~DefaultInputBitStream(); }; // Returns 1 or 0 inline int DefaultInputBitStream::readBit() { if (_availBits == 0) _availBits = pullCurrent(); // Triggers an exception if stream is closed _availBits--; return int(_current >> _availBits) & 1; } inline uint64 DefaultInputBitStream::readBits(uint count) { if ((count == 0) || (count > 64)) throw BitStreamException("Invalid bit count: " + TOSTR(count) + " (must be in [1..64])"); if (count <= _availBits) { _availBits -= count; return (_current >> _availBits) & (uint64(-1) >> (64 - count)); } // Not enough spots available in 'current' count -= _availBits; uint64 res = _current & ((uint64(1) << _availBits) - 1); _availBits = pullCurrent(); if (_availBits < count) throw BitStreamException("No more data to read in the bitstream", BitStreamException::END_OF_STREAM); _availBits -= count; const uint64 tail = (_current >> _availBits) & (uint64(-1) >> (64 - count)); return (count == 64) ? tail : ((res << count) | tail); } // Pull 64 bits of current value from buffer. inline uint DefaultInputBitStream::pullCurrent() { if (_position + 7 > _maxPosition) { if (_position > _maxPosition) readFromInputStream(_bufferSize); if (_position + 7 > _maxPosition) { // End of stream: overshoot max position => adjust bit index uint shift = uint(_maxPosition - _position) * 8; _availBits = shift + 8; uint64 val = 0; while (_position <= _maxPosition) { val |= (uint64(_buffer[_position++]) << shift); shift -= 8; } _current = val; return _availBits; } } // Regular processing, buffer length is multiple of 8 _current = uint64(BigEndian::readLong64(&_buffer[_position])); _position += 8; return 64; } #if !defined(_MSC_VER) || _MSC_VER > 1500 inline int64 DefaultInputBitStream::tell() { if (isClosed()) return -1; _is.clear(); const int64 res = int64(_is.tellg()); return (res < 0) ? -1 : 8 * (res - int64(_maxPosition + 1 - _position)) - int64(_availBits); } inline bool DefaultInputBitStream::seek(int64 pos) { if (isClosed()) return false; if (pos < 0) return false; // Update internal states to force read at new stream position _read += (8 * int64(_position) - int64(_availBits)); _availBits = 0; _position = 0; _maxPosition = -1; _is.clear(); _is.seekg(std::streampos(pos >> 3)); if (_is.fail()) return false; if ((pos & 7) != 0) readBits(pos & 7); return true; } #endif } #endif kanzi-cpp-2.6.0/src/bitstream/DefaultOutputBitStream.cpp000066400000000000000000000143511525576310500233350ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include "DefaultOutputBitStream.hpp" using namespace kanzi; using namespace std; DefaultOutputBitStream::DefaultOutputBitStream(OutputStream& os, uint bufferSize) : _os(os) { if (bufferSize < 1024) throw invalid_argument("Invalid buffer size (must be at least 1024)"); if (bufferSize > 1 << 29) throw invalid_argument("Invalid buffer size (must be at most 536870912)"); if ((bufferSize & 7) != 0) throw invalid_argument("Invalid buffer size (must be a multiple of 8)"); _availBits = 64; _bufferSize = bufferSize; _buffer = new kanzi::byte[_bufferSize]; _position = 0; _current = 0; _written = 0; _closed = false; memset(&_buffer[0], 0, size_t(_bufferSize)); } uint DefaultOutputBitStream::writeBits(const kanzi::byte bits[], uint count) { if (isClosed() == true) throw BitStreamException("Stream closed", BitStreamException::STREAM_CLOSED); uint remaining = count; uint start = 0; // Byte aligned cursor ? if ((_availBits & 7) == 0) { // Fill up _current while ((_availBits != 64) && (remaining >= 8)) { writeBits(uint64(bits[start]), 8); start++; remaining -= 8; } const uint maxPos = _bufferSize - 8; // Copy bits array to internal buffer while ((remaining >> 3) >= maxPos - _position) { memcpy(&_buffer[_position], &bits[start], maxPos - _position); start += (maxPos - _position); remaining -= ((maxPos - _position) << 3); _position = maxPos; flush(); } const uint r = (remaining >> 6) << 3; if (r > 0) { memcpy(&_buffer[_position], &bits[start], r); start += r; _position += r; remaining -= (r << 3); } } else if (remaining >= 64) { // Not kanzi::byte aligned const uint r = 64 - _availBits; const uint a = _availBits; while (remaining >= 256) { const uint64 v1 = uint64(BigEndian::readLong64(&bits[start])); const uint64 v2 = uint64(BigEndian::readLong64(&bits[start + 8])); const uint64 v3 = uint64(BigEndian::readLong64(&bits[start + 16])); const uint64 v4 = uint64(BigEndian::readLong64(&bits[start + 24])); _current |= (v1 >> r); if (_position >= _bufferSize - 32) flush(); BigEndian::writeLong64(&_buffer[_position], _current); BigEndian::writeLong64(&_buffer[_position + 8], (v1 << a) | (v2 >> r)); BigEndian::writeLong64(&_buffer[_position + 16], (v2 << a) | (v3 >> r)); BigEndian::writeLong64(&_buffer[_position + 24], (v3 << a) | (v4 >> r)); _position += 32; _current = (v4 << a); start += 32; remaining -= 256; _availBits = 64; } while (remaining >= 64) { const uint64 v = uint64(BigEndian::readLong64(&bits[start])); _current |= (v >> r); pushCurrent(); _current = v << a; start += 8; remaining -= 64; } _availBits = a; } // Last bytes while (remaining >= 8) { writeBits(uint64(bits[start]), 8); start++; remaining -= 8; } if (remaining > 0) writeBits(uint64(bits[start]) >> (8 - remaining), remaining); return count; } void DefaultOutputBitStream::_close() { if (isClosed() == true) return; uint savedBitIndex = _availBits; uint savedPosition = _position; uint64 savedCurrent = _current; try { // Push last bytes (the very last kanzi::byte may be incomplete) uint shift = 56; while (_availBits < 64) { _buffer[_position++] = kanzi::byte(_current >> shift); shift -= 8; _availBits += 8; } _written -= int64(_availBits - 64); // can be negative _availBits = 64; flush(); } catch (const BitStreamException&) { // Revert fields to allow subsequent attempts in case of transient failure _position = savedPosition; _availBits = savedBitIndex; _current = savedCurrent; throw; // re-throw } try { _os.flush(); if (_os.fail()) throw BitStreamException("Write to bitstream failed.", BitStreamException::INPUT_OUTPUT); } catch (const ios_base::failure& e) { throw BitStreamException(e.what(), BitStreamException::INPUT_OUTPUT); } _closed = true; _position = 0; _availBits = 0; _written -= 64; // adjust because _availBits = 0 // Reset fields to force a flush() and trigger an exception // on writeBit() or writeBits() kanzi::byte* buffer = new kanzi::byte[8]; memset(&buffer[0], 0, size_t(8)); delete[] _buffer; _buffer = buffer; _bufferSize = 8; } // Write buffer to underlying stream void DefaultOutputBitStream::flush() { if (isClosed() == true) throw BitStreamException("Stream closed", BitStreamException::STREAM_CLOSED); try { if (_position > 0) { _os.write(reinterpret_cast(_buffer), _position); if (_os.fail()) throw BitStreamException("Write to bitstream failed", BitStreamException::INPUT_OUTPUT); _written += (int64(_position) << 3); _position = 0; } } catch (const ios_base::failure& e) { throw BitStreamException(e.what(), BitStreamException::INPUT_OUTPUT); } } DefaultOutputBitStream::~DefaultOutputBitStream() { try { _close(); } catch (const exception&) { // Ignore and continue } delete[] _buffer; } kanzi-cpp-2.6.0/src/bitstream/DefaultOutputBitStream.hpp000066400000000000000000000112641525576310500233420ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_DefaultOutputBitStream #define knz_DefaultOutputBitStream #include "../BitStreamException.hpp" #include "../OutputStream.hpp" #include "../OutputBitStream.hpp" #include "../Memory.hpp" #include "../Seekable.hpp" #include "../util/strings.hpp" namespace kanzi { #if defined(_MSC_VER) && _MSC_VER <= 1500 class DefaultOutputBitStream FINAL : public OutputBitStream #else class DefaultOutputBitStream FINAL : public OutputBitStream, public Seekable #endif { private: OutputStream& _os; byte* _buffer; bool _closed; uint _bufferSize; uint _position; // index of current byte in buffer uint _availBits; // bits not consumed in _current int64 _written; uint64 _current; // cached bits void pushCurrent(); void flush(); void _close(); public: DefaultOutputBitStream(OutputStream& os, uint bufferSize=65536); ~DefaultOutputBitStream(); void writeBit(int bit); uint writeBits(uint64 bits, uint length); uint writeBits(const byte bits[], uint length); void close() { _close(); } #if !defined(_MSC_VER) || _MSC_VER > 1500 int64 tell(); bool seek(int64 pos); #endif // Return number of bits written so far uint64 written() const { // Number of bits flushed + bytes written in memory + bits written in memory return uint64(_written + (int64(_position) << 3) + int64(64 - _availBits)); } bool isClosed() const { return _closed; } }; // Write least significant bit of the input integer. Trigger exception if stream is closed inline void DefaultOutputBitStream::writeBit(int bit) { if (_availBits <= 1) { // _availBits = 0 if stream is closed => force pushCurrent() _current |= (uint64(bit) & 1); pushCurrent(); } else { _availBits--; _current |= (uint64(bit & 1) << _availBits); } } // Write 'count' (in [0..64]) bits. A zero count is a successful no-op, // including when the stream is closed. Otherwise, trigger an exception if closed. inline uint DefaultOutputBitStream::writeBits(uint64 value, uint count) { if ((count == 0) || (count > 64)) return 0; if (count < _availBits) { _availBits -= count; _current |= (value << _availBits); } else { // Not enough spots available in 'current' const uint remaining = count - _availBits; _current |= (_availBits == 0 ? 0 : (value >> remaining) & (~uint64(0) >> (64 - _availBits))); pushCurrent(); if (remaining != 0) { _availBits -= remaining; _current = value << _availBits; } } return count; } // Push 64 bits of current value into buffer. inline void DefaultOutputBitStream::pushCurrent() { BigEndian::writeLong64(&_buffer[_position], int64(_current)); _availBits = 64; _current = 0; _position += 8; if (_position >= _bufferSize - 8) flush(); } #if !defined(_MSC_VER) || _MSC_VER > 1500 inline int64 DefaultOutputBitStream::tell() { if (isClosed() == true) return -1; _os.clear(); const int64 res = int64(_os.tellp()); return (res < 0) ? -1 : 8 * res + (int64(_position) << 3) + int64(64 - _availBits); } // Only support a new position at the byte boundary (pos & 7 == 0) inline bool DefaultOutputBitStream::seek(int64 pos) { if (isClosed() == true) return false; if ((pos < 0) || ((pos & 7) != 0)) return false; // Flush buffer // Round down to byte alignment const uint a = _availBits - (_availBits & 7); for (int i = 56; i >= int(a); i -= 8) { _buffer[_position++] = byte(_current >> uint(i)); if (_position >= _bufferSize) flush(); } _availBits = 64; _current = 0; flush(); _os.clear(); _os.seekp(std::streampos(pos >> 3)); return _os.fail() == false; } #endif } #endif kanzi-cpp-2.6.0/src/concurrent.hpp000066400000000000000000000164661525576310500171230ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_concurrent #define knz_concurrent #include "types.hpp" #if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1700) // C++ 11 (or partial) #include #define HAVE_STD_ATOMICS 1 #ifndef CONCURRENCY_DISABLED #ifdef __clang__ // Process clang first because it may define __GNUC__ with an old version #define CONCURRENCY_ENABLED #elif __GNUC__ // Require g++ 5.0 minimum, 4.8.4 generates exceptions on futures (?) #if ((__GNUC__ << 16) + __GNUC_MINOR__ >= (5 << 16) + 0) #define CONCURRENCY_ENABLED #endif #else #define CONCURRENCY_ENABLED #endif #endif #else #define HAVE_STD_ATOMICS 0 #endif #ifdef CONCURRENCY_ENABLED #include #include #include #include #include #include #include #include #if __cplusplus >= 201703L #include #endif #include #ifdef __x86_64__ #ifdef __clang__ #define CPU_PAUSE() __builtin_ia32_pause() #elif __GNUC__ #define CPU_PAUSE() __builtin_ia32_pause() #elif _MSC_VER #include #define CPU_PAUSE() _mm_pause() #else #define CPU_PAUSE() std::this_thread::yield(); #endif #else #define CPU_PAUSE() std::this_thread::yield(); #endif #else #define CPU_PAUSE() #endif template class Task { public: Task() {} virtual ~Task() {} virtual T run() = 0; }; #ifdef CONCURRENCY_ENABLED class ThreadPool FINAL { public: ThreadPool(int threads = 8); template #if __cplusplus >= 201703L // result_of deprecated from C++17 std::future> schedule(F&& f, Args&&... args); #else std::future::type> schedule(F&& f, Args&&... args); #endif ~ThreadPool() noexcept; private: std::vector _workers; std::queue> _tasks; std::mutex _mutex; std::condition_variable _condition; bool _stop; }; inline ThreadPool::ThreadPool(int threads) : _stop(false) { if ((threads <= 0) || (threads > 1024)) throw std::invalid_argument("The number of threads must be in [1..1024]"); // Start and run threads for (int i = 0; i < threads; i++) _workers.emplace_back( [this] { for(;;) { std::function task; { std::unique_lock lock(_mutex); _condition.wait(lock, [this] { return _stop || !_tasks.empty(); }); if (_stop && _tasks.empty()) return; task = std::move(_tasks.front()); _tasks.pop(); } task(); } } ); } template #if __cplusplus >= 201703L // result_of deprecated from C++17 std::future > ThreadPool::schedule(F&& f, Args&&... args) { using return_type = typename std::invoke_result::type; #else std::future::type> ThreadPool::schedule(F&& f, Args&&... args) { using return_type = typename std::result_of::type; #endif #if __cplusplus >= 201703L auto task = std::make_shared>( [fn = std::forward(f), params = std::make_tuple(std::forward(args)...)]() mutable -> return_type { return std::apply(std::move(fn), std::move(params)); } ); #else auto task = std::make_shared>( std::bind(std::forward(f), std::forward(args)...) ); #endif std::future res = task->get_future(); { std::unique_lock lock(_mutex); if (_stop == true) throw std::runtime_error("ThreadPool stopped"); _tasks.emplace([task](){ (*task)(); }); } _condition.notify_one(); return res; } // the destructor joins all threads inline ThreadPool::~ThreadPool() noexcept { { std::unique_lock lock(_mutex); _stop = true; } _condition.notify_all(); for (std::thread& w : _workers) w.join(); } template class BoundedConcurrentQueue { public: BoundedConcurrentQueue(int nbItems, T* data) : _index(0), _size(nbItems), _data(data) {} ~BoundedConcurrentQueue() { } T* get() { int idx = _index.fetch_add(1, std::memory_order_acq_rel); return (idx >= _size) ? nullptr : &_data[idx]; } void clear() { _index.store(_size); } private: std::atomic_int _index; int _size; T* _data; }; #endif #if HAVE_STD_ATOMICS typedef std::atomic_int atomic_int_t; #define LOAD_ATOMIC(a) ((a).load(std::memory_order_acquire)) #define STORE_ATOMIC(a, v) ((a).store((v), std::memory_order_release)) #define EXCHANGE_ATOMIC(a, v) ((a).exchange((v), std::memory_order_acq_rel)) #define FETCH_ADD_ATOMIC(a, v) ((a).fetch_add((v), std::memory_order_acq_rel)) #define COMPARE_EXCHANGE_ATOMIC(obj, expected, desired) \ ((obj).compare_exchange_strong((expected), (desired), \ std::memory_order_release, std::memory_order_acquire)) #else typedef int atomic_int_t; #define LOAD_ATOMIC(a) (a) #define STORE_ATOMIC(a, v) ((a) = (v)) #define EXCHANGE_ATOMIC(a, v) exchange_atomic_int((a), (v)) #define FETCH_ADD_ATOMIC(a, v) fetch_add_atomic_int((a), (v)) #define COMPARE_EXCHANGE_ATOMIC(obj, expected, desired) \ compare_exchange_fallback((obj), (expected), (desired)) inline int exchange_atomic_int(int& a, int v) { int old = a; a = v; return old; } inline int fetch_add_atomic_int(int& a, int v) { int old = a; a += v; return old; } inline bool compare_exchange_fallback(int& obj, int& expected, int desired) { if (obj == expected) { obj = desired; return true; } else { expected = obj; // update expected on failure return false; } } #endif #endif kanzi-cpp-2.6.0/src/configure000077500000000000000000000000231525576310500161150ustar00rootroot00000000000000# Fake config file kanzi-cpp-2.6.0/src/entropy/000077500000000000000000000000001525576310500157135ustar00rootroot00000000000000kanzi-cpp-2.6.0/src/entropy/ANSRangeDecoder.cpp000066400000000000000000000255301525576310500213100ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include "../BitStreamException.hpp" #include "../Global.hpp" #include "ANSRangeDecoder.hpp" #include "EntropyUtils.hpp" using namespace kanzi; using namespace std; const uint ANSRangeDecoder::ANS_TOP = 1 << 15; // max possible for ANS_TOP=1<<23 const int ANSRangeDecoder::DEFAULT_ANS0_CHUNK_SIZE = 16384; const int ANSRangeDecoder::DEFAULT_LOG_RANGE = 12; const int ANSRangeDecoder::MIN_CHUNK_SIZE = 1024; const int ANSRangeDecoder::MAX_CHUNK_SIZE = 1 << 27; // 8*MAX_CHUNK_SIZE must not over static const int ANS_DECODE_CHECK_INTERVAL = 128; static const uint ANS_PAYLOAD_GUARD = 2 * ANS_DECODE_CHECK_INTERVAL + 2; // The chunk size indicates how many bytes are encoded (per block) before // resetting the frequency stats. ANSRangeDecoder::ANSRangeDecoder(InputBitStream& bitstream, int order, int chunkSize) : _bitstream(bitstream) { if ((order != 0) && (order != 1)) throw invalid_argument("ANS Codec: The order must be 0 or 1"); if (chunkSize < MIN_CHUNK_SIZE) { stringstream ss; ss << "ANS Codec: The chunk size must be at least " << MIN_CHUNK_SIZE; throw invalid_argument(ss.str()); } if (chunkSize > MAX_CHUNK_SIZE) { stringstream ss; ss << "ANS Codec: The chunk size must be at most " << MAX_CHUNK_SIZE; throw invalid_argument(ss.str()); } const uint64 scaledChunkSize = uint64(chunkSize) << (8 * order); _chunkSize = uint(min(scaledChunkSize, uint64(MAX_CHUNK_SIZE))); _order = order; const int dim = 255 * order + 1; _freqs = new uint[dim * 256]; _symbols = new ANSDecSymbol[dim * 256]; _buffer = nullptr; _bufferSize = 0; _f2s = nullptr; _f2sSize = 0; _logRange = DEFAULT_LOG_RANGE; } ANSRangeDecoder::~ANSRangeDecoder() { _dispose(); if (_buffer != nullptr) delete[] _buffer; if (_f2s != nullptr) delete[] _f2s; delete[] _freqs; delete[] _symbols; } int ANSRangeDecoder::decodeHeader(uint frequencies[], uint alphabet[]) { _logRange = int(8 + _bitstream.readBits(3)); if (_logRange > 15) { stringstream ss; ss << "Invalid bitstream: range = " << _logRange << " (must be in [8..15])"; throw BitStreamException(ss.str(), BitStreamException::INVALID_STREAM); } int res = 0; const int dim = 255 * _order + 1; if (_f2sSize < (dim << _logRange)) { const int newSize = dim << _logRange; uint8* f2s = new uint8[newSize]; delete[] _f2s; _f2s = f2s; _f2sSize = newSize; } const uint scale = 1 << _logRange; const int llr = Global::_log2(_logRange) + 1; for (int k = 0; k < dim; k++) { const int alphabetSize = EntropyUtils::decodeAlphabet(_bitstream, alphabet); if (alphabetSize == 0) { if ((_order == 1) && (k == 0)) throw BitStreamException("Invalid bitstream: missing ANS1 context 0", BitStreamException::INVALID_STREAM); if (_order == 1) { // Empty ANS1 alphabets implicitly model the singleton {0}. memset(&_f2s[k << _logRange], 0, size_t(scale)); _symbols[k << 8].reset(0, scale, _logRange); } continue; } uint* f = &frequencies[k << 8]; if (alphabetSize != 256) memset(f, 0, sizeof(uint) * 256); const int chkSize = (alphabetSize >= 64) ? 8 : 6; uint sum = 0; // Decode all frequencies (but the first one) by chunks for (int i = 1; i < alphabetSize; i += chkSize) { // Read frequencies size for current chunk const uint logMax = uint(_bitstream.readBits(llr)); if (logMax > _logRange) { stringstream ss; ss << "Invalid bitstream: incorrect frequency size "; ss << logMax << " in ANS range decoder"; throw BitStreamException(ss.str(), BitStreamException::INVALID_STREAM); } const int endj = min(i + chkSize, alphabetSize); // Read frequencies for (int j = i; j < endj; j++) { const uint freq = (logMax == 0) ? 1 : uint(_bitstream.readBits(logMax) + 1); if (freq >= scale) { stringstream ss; ss << "Invalid bitstream: incorrect frequency " << freq; ss << " for symbol '" << alphabet[j] << "' in ANS range decoder"; throw BitStreamException(ss.str(), BitStreamException::INVALID_STREAM); } f[alphabet[j]] = freq; sum += freq; } } // Infer first frequency if (scale <= sum) { stringstream ss; ss << "Invalid bitstream: incorrect frequency " << frequencies[alphabet[0]]; ss << " for symbol '" << alphabet[0] << "' in ANS range decoder"; throw BitStreamException(ss.str(), BitStreamException::INVALID_STREAM); } f[alphabet[0]] = uint(scale - sum); sum = 0; ANSDecSymbol* symb = &_symbols[k << 8]; uint8* freq2sym = &_f2s[k << _logRange]; // Create reverse mapping for (int i = 0; i < 256; i++) { if (f[i] == 0) continue; memset(&freq2sym[sum], i, size_t(f[i])); symb[i].reset(sum, f[i], _logRange); sum += f[i]; } res += alphabetSize; } return res; } int ANSRangeDecoder::decode(kanzi::byte block[], uint blkptr, uint count) { if (count <= 32) { _bitstream.readBits(&block[blkptr], 8 * count); return count; } const uint size = min(uint(_chunkSize), count); const uint extra = max(size >> 3, min(size, uint(1 << 16))); const uint minBufSize = size + extra + ANS_PAYLOAD_GUARD; if (_bufferSize < minBufSize) { kanzi::byte* buffer = new kanzi::byte[minBufSize]; delete[] _buffer; _buffer = buffer; _bufferSize = minBufSize; } const uint end = blkptr + count; uint startChunk = blkptr; uint alphabet[256]; while (startChunk < end) { const uint sizeChunk = min(uint(_chunkSize), end - startChunk); const int alphabetSize = decodeHeader(_freqs, alphabet); if (alphabetSize == 0) return startChunk - blkptr; if ((_order == 0) && (alphabetSize == 1)) { // Shortcut for chunks with only one symbol memset(&block[startChunk], alphabet[0], size_t(sizeChunk)); } else { if (decodeChunk(&block[startChunk], sizeChunk) == false) return -1; } startChunk += sizeChunk; } return count; } bool ANSRangeDecoder::decodeChunk(kanzi::byte block[], uint count) { // Read chunk size const uint sz = uint(EntropyUtils::readVarInt(_bitstream)); if ((sz >= MAX_CHUNK_SIZE) || (sz > _bufferSize - ANS_PAYLOAD_GUARD)) return false; // Read initial ANS states uint st0 = uint(_bitstream.readBits(32)); uint st1 = uint(_bitstream.readBits(32)); uint st2 = uint(_bitstream.readBits(32)); uint st3 = uint(_bitstream.readBits(32)); if (count == 0) return true; // Read encoded data from bitstream. decodeSymbol() is branchless and may // speculatively read two bytes when no renormalization is needed. Keep a // small, zeroed guard after the declared payload and validate consumption // in batches below. _bitstream.readBits(&_buffer[0], 8 * sz); memset(&_buffer[sz], 0, ANS_PAYLOAD_GUARD); kanzi::byte* p = &_buffer[0]; kanzi::byte* const endPayload = &_buffer[sz]; const int mask = (1 << _logRange) - 1; const int count4 = count & -4; if (_order == 0) { for (int batch = 0; batch < count4; batch += ANS_DECODE_CHECK_INTERVAL) { const int endBatch = min(batch + ANS_DECODE_CHECK_INTERVAL, count4); for (int i = batch; i < endBatch; i += 4) { const uint8 cur3 = _f2s[st3 & mask]; block[i] = kanzi::byte(cur3); st3 = decodeSymbol(p, st3, _symbols[cur3], mask); const uint8 cur2 = _f2s[st2 & mask]; block[i + 1] = kanzi::byte(cur2); st2 = decodeSymbol(p, st2, _symbols[cur2], mask); const uint8 cur1 = _f2s[st1 & mask]; block[i + 2] = kanzi::byte(cur1); st1 = decodeSymbol(p, st1, _symbols[cur1], mask); const uint8 cur0 = _f2s[st0 & mask]; block[i + 3] = kanzi::byte(cur0); st0 = decodeSymbol(p, st0, _symbols[cur0], mask); } if (p > endPayload) return false; } } else { const int quarter = count4 >> 2; int i0 = 0; int i1 = 1 * quarter; int i2 = 2 * quarter; int i3 = 3 * quarter; int prv0 = 0, prv1 = 0, prv2 = 0, prv3 = 0; for (int batch = 0; batch < quarter; batch += ANS_DECODE_CHECK_INTERVAL >> 2) { const int endBatch = min(batch + (ANS_DECODE_CHECK_INTERVAL >> 2), quarter); for ( ; i0 < endBatch; i0++, i1++, i2++, i3++) { const uint8 cur3 = _f2s[(prv3 << _logRange) + (st3 & mask)]; const uint8 cur2 = _f2s[(prv2 << _logRange) + (st2 & mask)]; const uint8 cur1 = _f2s[(prv1 << _logRange) + (st1 & mask)]; const uint8 cur0 = _f2s[(prv0 << _logRange) + (st0 & mask)]; st3 = decodeSymbol(p, st3, _symbols[(prv3 << 8) | cur3], mask); st2 = decodeSymbol(p, st2, _symbols[(prv2 << 8) | cur2], mask); st1 = decodeSymbol(p, st1, _symbols[(prv1 << 8) | cur1], mask); st0 = decodeSymbol(p, st0, _symbols[(prv0 << 8) | cur0], mask); block[i3] = kanzi::byte(cur3); block[i2] = kanzi::byte(cur2); block[i1] = kanzi::byte(cur1); block[i0] = kanzi::byte(cur0); prv3 = cur3; prv2 = cur2; prv1 = cur1; prv0 = cur0; } if (p > endPayload) return false; } } const uint tail = count - uint(count4); if (size_t(endPayload - p) < tail) return false; for (uint i = count4; i < count; i++) block[i] = *p++; return p == endPayload; } kanzi-cpp-2.6.0/src/entropy/ANSRangeDecoder.hpp000066400000000000000000000056521525576310500213200ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_ANSRangeDecoder #define knz_ANSRangeDecoder #include "../EntropyDecoder.hpp" #include "../types.hpp" // Implementation of an Asymmetric Numeral System decoder. // See "Asymmetric Numeral System" by Jarek Duda at http://arxiv.org/abs/0902.0271 // Some code has been ported from https://github.com/rygorous/ryg_rans // For an alternate C implementation example, see https://github.com/Cyan4973/FiniteStateEntropy namespace kanzi { struct ANSDecSymbol { void reset(int cumFreq, int freq, int logRange); uint16 _cumFreq; uint16 _freq; }; class ANSRangeDecoder : public EntropyDecoder { public: static const uint ANS_TOP; ANSRangeDecoder(InputBitStream& bitstream, int order = 0, int chunkSize = DEFAULT_ANS0_CHUNK_SIZE); ~ANSRangeDecoder(); int decode(byte block[], uint blkptr, uint len); InputBitStream& getBitStream() const { return _bitstream; } void dispose() { _dispose(); } private: static const int DEFAULT_ANS0_CHUNK_SIZE; static const int DEFAULT_LOG_RANGE; static const int MIN_CHUNK_SIZE; static const int MAX_CHUNK_SIZE; InputBitStream& _bitstream; uint* _freqs; uint8* _f2s; int _f2sSize; ANSDecSymbol* _symbols; byte* _buffer; uint _bufferSize; uint _chunkSize; uint _order; uint _logRange; bool decodeChunk(byte block[], uint count); uint decodeSymbol(byte*& p, uint& st, const ANSDecSymbol& sym, const int mask) const; int decodeHeader(uint frequencies[], uint alphabet[]); void _dispose() const {} }; inline void ANSDecSymbol::reset(int cumFreq, int freq, int logRange) { _cumFreq = uint16(cumFreq); _freq = (freq >= (1 << logRange)) ? uint16((1 << logRange) - 1) : uint16(freq); // Mirror encoder } inline uint ANSRangeDecoder::decodeSymbol(byte*& p, uint& st, const ANSDecSymbol& sym, const int mask) const { // Compute next ANS state // D(x) = (s, q_s (x/M) + mod(x,M) - b_s) where s is such b_s <= x mod M < b_{s+1} st = uint(sym._freq) * (st >> _logRange) + (st & mask) - uint(sym._cumFreq); // Normalize const int x = (st < ANS_TOP) ? -1 : 0; st = (st << (x & 16)) | (x & ((uint(p[0]) << 8) | uint(p[1]))); p -= (x + x); return st; } } #endif kanzi-cpp-2.6.0/src/entropy/ANSRangeEncoder.cpp000066400000000000000000000222741525576310500213240ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include "ANSRangeEncoder.hpp" #include "EntropyUtils.hpp" #include "../Global.hpp" #include "../Memory.hpp" using namespace kanzi; using namespace std; const int ANSRangeEncoder::ANS_TOP = 1 << 15; // max possible for ANS_TOP=1<<23 const int ANSRangeEncoder::DEFAULT_ANS0_CHUNK_SIZE = 16384; const int ANSRangeEncoder::DEFAULT_LOG_RANGE = 12; const int ANSRangeEncoder::MIN_CHUNK_SIZE = 1024; const int ANSRangeEncoder::MAX_CHUNK_SIZE = 1 << 27; // 8*MAX_CHUNK_SIZE must not overflow // The chunk size indicates how many bytes are encoded (per block) before // resetting the frequency stats. ANSRangeEncoder::ANSRangeEncoder(OutputBitStream& bitstream, int order, int chunkSize, int logRange) : _bitstream(bitstream) { if ((order != 0) && (order != 1)) throw invalid_argument("ANS Codec: The order must be 0 or 1"); if (chunkSize < MIN_CHUNK_SIZE) { stringstream ss; ss << "ANS Codec: The chunk size must be at least " << MIN_CHUNK_SIZE; throw invalid_argument(ss.str()); } if (chunkSize > MAX_CHUNK_SIZE) { stringstream ss; ss << "ANS Codec: The chunk size must be at most " << MAX_CHUNK_SIZE; throw invalid_argument(ss.str()); } if ((logRange < 8) || (logRange > 15)) { stringstream ss; ss << "ANS Codec: Invalid range: " << logRange << " (must be in [8..15])"; throw invalid_argument(ss.str()); } const uint64 scaledChunkSize = uint64(chunkSize) << (8 * order); _chunkSize = uint(min(scaledChunkSize, uint64(MAX_CHUNK_SIZE))); _order = order; const int dim = 255 * order + 1; _symbols = new ANSEncSymbol[dim * 256]; _freqs = new uint[dim * 257]; // freqs[x][256] = total(freqs[x][0..255]) _buffer = nullptr; _bufferSize = 0; _logRange = (order == 0) ? logRange : max(logRange - 1, 8); } ANSRangeEncoder::~ANSRangeEncoder() { _dispose(); if (_buffer != nullptr) delete[] _buffer; delete[] _symbols; delete[] _freqs; } // Compute cumulated frequencies and encode header int ANSRangeEncoder::updateFrequencies(uint frequencies[], uint lr) { int res = 0; const int endk = 255 * _order + 1; _bitstream.writeBits(lr - 8, 3); // logRange uint curAlphabet[256]; for (int k = 0; k < endk; k++) { uint* f = &frequencies[k * 257]; const int alphabetSize = EntropyUtils::normalizeFrequencies(f, curAlphabet, 256, f[256], 1 << lr); if (alphabetSize > 0) { ANSEncSymbol* symb = &_symbols[k << 8]; int sum = 0; for (int i = 0, count = 0; i < 256; i++) { if (f[i] == 0) continue; symb[i].reset(sum, f[i], lr); sum += f[i]; count++; if (count >= alphabetSize) break; } } if ((_order == 1) && (alphabetSize == 0)) { // The empty alphabet marker implicitly models the singleton {0}. _symbols[k << 8].reset(0, 1 << lr, lr); } encodeHeader(alphabetSize, curAlphabet, f, lr); res += alphabetSize; } return res; } // Encode alphabet and frequencies bool ANSRangeEncoder::encodeHeader(int alphabetSize, const uint alphabet[], const uint frequencies[], uint lr) const { const int encoded = EntropyUtils::encodeAlphabet(_bitstream, alphabet, 256, alphabetSize); if (encoded < 0) return false; if (encoded <= 1) return true; const int chkSize = (alphabetSize >= 64) ? 8 : 6; const int llr = Global::_log2(lr) + 1; // Encode all frequencies (but the first one) by chunks for (int i = 1; i < alphabetSize; i += chkSize) { uint max = frequencies[alphabet[i]] - 1; const int endj = min(i + chkSize, alphabetSize); // Search for max frequency log size in next chunk for (int j = i + 1; j < endj; j++) { if (frequencies[alphabet[j]] - 1 > max) max = frequencies[alphabet[j]] - 1; } const uint logMax = (max == 0) ? 0 : Global::_log2(max) + 1; _bitstream.writeBits(logMax, llr); if (logMax == 0) // all frequencies equal one in this chunk continue; // Write frequencies for (int j = i; j < endj; j++) _bitstream.writeBits(frequencies[alphabet[j]] - 1, logMax); } return true; } // Dynamically compute the frequencies for every chunk of data in the block int ANSRangeEncoder::encode(const kanzi::byte block[], uint blkptr, uint count) { if (count <= 32) { _bitstream.writeBits(&block[blkptr], 8 * count); return count; } const uint end = blkptr + count; uint startChunk = blkptr; uint sz = uint(_chunkSize); const uint size = min(sz, count); const uint extra = max(size >> 3, min(size, uint(1 << 16))); const uint bufSize = size + extra; if (_bufferSize < bufSize) { kanzi::byte* buffer = new kanzi::byte[bufSize]; delete[] _buffer; _buffer = buffer; _bufferSize = bufSize; } while (startChunk < end) { const uint sizeChunk = min(sz, end - startChunk); const int alphabetSize = rebuildStatistics(&block[startChunk], sizeChunk, _logRange); // Skip chunk if only one symbol if ((alphabetSize <= 1) && (_order == 0)) { startChunk += sizeChunk; continue; } encodeChunk(&block[startChunk], sizeChunk); startChunk += sizeChunk; } return count; } void ANSRangeEncoder::encodeChunk(const kanzi::byte block[], int end) { int st0 = ANS_TOP; int st1 = ANS_TOP; int st2 = ANS_TOP; int st3 = ANS_TOP; kanzi::byte* p = &_buffer[_bufferSize - 1]; const kanzi::byte* p0 = p; const int end4 = end & -4; for (int i = end - 1; i >= end4; i--) *p-- = block[i]; if (_order == 0) { for (int i = end4 - 1; i > 0; i -= 4) { st0 = encodeSymbol(p, st0, _symbols[int(block[i])]); st1 = encodeSymbol(p, st1, _symbols[int(block[i - 1])]); st2 = encodeSymbol(p, st2, _symbols[int(block[i - 2])]); st3 = encodeSymbol(p, st3, _symbols[int(block[i - 3])]); } } else { // order 1 const int quarter = end4 >> 2; int i0 = 1 * quarter - 2; int i1 = 2 * quarter - 2; int i2 = 3 * quarter - 2; int i3 = end4 - 2; int prv0 = int(block[i0 + 1]); int prv1 = int(block[i1 + 1]); int prv2 = int(block[i2 + 1]); int prv3 = int(block[i3 + 1]); for ( ; i0 >= 0; i0--, i1--, i2--, i3--) { const int cur0 = int(block[i0]); st0 = encodeSymbol(p, st0, _symbols[(cur0 << 8) | prv0]); const int cur1 = int(block[i1]); st1 = encodeSymbol(p, st1, _symbols[(cur1 << 8) | prv1]); const int cur2 = int(block[i2]); st2 = encodeSymbol(p, st2, _symbols[(cur2 << 8) | prv2]); const int cur3 = int(block[i3]); st3 = encodeSymbol(p, st3, _symbols[(cur3 << 8) | prv3]); prv0 = cur0; prv1 = cur1; prv2 = cur2; prv3 = cur3; } // Last symbols st0 = encodeSymbol(p, st0, _symbols[prv0]); st1 = encodeSymbol(p, st1, _symbols[prv1]); st2 = encodeSymbol(p, st2, _symbols[prv2]); st3 = encodeSymbol(p, st3, _symbols[prv3]); } // Write chunk size EntropyUtils::writeVarInt(_bitstream, uint32(p0 - p)); // Write final ANS states _bitstream.writeBits(st0, 32); _bitstream.writeBits(st1, 32); _bitstream.writeBits(st2, 32); _bitstream.writeBits(st3, 32); if (p != p0) { // Write encoded data to bitstream _bitstream.writeBits(&p[1], 8 * uint(p0 - p)); } } // Compute chunk frequencies, cumulated frequencies and encode chunk header int ANSRangeEncoder::rebuildStatistics(const kanzi::byte block[], int end, uint lr) { const int dim = 255 * _order + 1; memset(_freqs, 0, size_t(257 * dim) * sizeof(uint)); if (_order == 0){ Global::computeHistogram(block, end, _freqs, true, true); } else { const int quarter = end >> 2; if (quarter == 0) { Global::computeHistogram(block, end, _freqs, false, true); } else { Global::computeHistogram(&block[0 * quarter], quarter, _freqs, false, true); Global::computeHistogram(&block[1 * quarter], quarter, _freqs, false, true); Global::computeHistogram(&block[2 * quarter], quarter, _freqs, false, true); Global::computeHistogram(&block[3 * quarter], quarter, _freqs, false, true); } } return updateFrequencies(_freqs, lr); } kanzi-cpp-2.6.0/src/entropy/ANSRangeEncoder.hpp000066400000000000000000000076541525576310500213360ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_ANSRangeEncoder #define knz_ANSRangeEncoder #include "../EntropyEncoder.hpp" // Implementation of an Asymmetric Numeral System encoder. // See "Asymmetric Numeral System" by Jarek Duda at http://arxiv.org/abs/0902.0271 // Some code has been ported from https://github.com/rygorous/ryg_rans // For an alternate C implementation example, see https://github.com/Cyan4973/FiniteStateEntropy namespace kanzi { struct ANSEncSymbol { void reset(int cumFreq, int freq, uint logRange); int _xMax; // (Exclusive) upper bound of pre-normalization interval int _bias; // Bias int _cmplFreq; // Complement of frequency: (1 << scale_bits) - freq int _invShift; // Reciprocal shift uint64 _invFreq; // Fixed-point reciprocal frequency }; class ANSRangeEncoder : public EntropyEncoder { public: static const int ANS_TOP; ANSRangeEncoder(OutputBitStream& bitstream, int order = 0, int chunkSize = DEFAULT_ANS0_CHUNK_SIZE, int logRange = DEFAULT_LOG_RANGE); ~ANSRangeEncoder(); int updateFrequencies(uint frequencies[], uint lr); int encode(const byte block[], uint blkptr, uint len); OutputBitStream& getBitStream() const { return _bitstream; } void dispose() { _dispose(); } private: static const int DEFAULT_ANS0_CHUNK_SIZE; static const int DEFAULT_LOG_RANGE; static const int MIN_CHUNK_SIZE; static const int MAX_CHUNK_SIZE; ANSEncSymbol* _symbols; uint* _freqs; byte* _buffer; uint _bufferSize; OutputBitStream& _bitstream; uint _chunkSize; uint _logRange; uint _order; int rebuildStatistics(const byte block[], int end, uint lr); void encodeChunk(const byte block[], int end); int encodeSymbol(byte*& p, int& st, const ANSEncSymbol& sym) const; bool encodeHeader(int alphabetSize, const uint alphabet[], const uint frequencies[], uint lr) const; void _dispose() const {} }; inline void ANSEncSymbol::reset(int cumFreq, int freq, uint logRange) { // Make sure xMax is a positive int32. Compatibility with Java implementation if (freq >= 1 << logRange) freq = (1 << logRange) - 1; _xMax = ((ANSRangeEncoder::ANS_TOP >> logRange) << 16) * freq; _cmplFreq = (1 << logRange) - freq; if (freq < 2) { _invFreq = uint64(0xFFFFFFFF); _invShift = 32; _bias = cumFreq + (1 << logRange) - 1; } else { int shift = 0; while (freq > (1 << shift)) shift++; // Alverson, "Integer Division using reciprocals" _invFreq = (((uint64(1) << (shift + 31)) + freq - 1) / freq) & uint64(0xFFFFFFFF); _invShift = 32 + shift - 1; _bias = cumFreq; } } inline int ANSRangeEncoder::encodeSymbol(byte*& p, int& st, const ANSEncSymbol& sym) const { const int x = (st >= sym._xMax) ? 1 : 0; *p = byte(st); p -= x; *p = byte(st >> 8); p -= x; st >>= (-x & 16); // Compute next ANS state // C(s,x) = M floor(x/q_s) + mod(x,q_s) + b_s where b_s = q_0 + ... + q_{s-1} // st = ((st / freq) << lr) + (st % freq) + cumFreq; return st + sym._bias + int((st * sym._invFreq) >> sym._invShift) * sym._cmplFreq; } } #endif kanzi-cpp-2.6.0/src/entropy/AdaptiveProbMap.hpp000066400000000000000000000103431525576310500214430ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_AdaptiveProbMap #define knz_AdaptiveProbMap #include "../Global.hpp" // APM maps a probability and a context into a new probability // that the next bit will be 1. After each guess, it updates // its state to improve future guesses. namespace kanzi { template class LinearAdaptiveProbMap { public: LinearAdaptiveProbMap(int n); ~LinearAdaptiveProbMap() { delete[] _data; } int get(int bit, int pr, int ctx); private: int _index; // last p, context uint16* _data; // [NbCtx][33]: p, context -> p }; template inline LinearAdaptiveProbMap::LinearAdaptiveProbMap(int n) { const int size = (n == 0) ? 65 : n * 65; _data = new uint16[size]; _index = 0; for (int j = 0; j <= 64; j++) { // 65536 does not fit in uint16. Keep the endpoint aligned // with the maximum target used by the adaptive update. _data[j] = (j == 64) ? uint16(65528) : uint16(j << 10); } for (int i = 1; i < n; i++) { memcpy(&_data[i * 65], &_data[0], 65 * sizeof(uint16)); } } // Return improved prediction given current bit, prediction and context template inline int LinearAdaptiveProbMap::get(int bit, int pr, int ctx) { // Update probability based on error and learning rate const int g = -bit & 65528; _data[_index] += (((g - int(_data[_index])) >> RATE) + bit); _data[_index + 1] += (((g - int(_data[_index + 1])) >> RATE) + bit); // Find index: 65*ctx + quantized prediction in [0..64] _index = (pr >> 6) + 65 * ctx; // Return interpolated probability. The table index advances every 64 // prediction units, so use the matching 6-bit fractional part. const uint16 w = uint16(pr & 63); return ((_data[_index] << 6) + (int(_data[_index + 1]) - int(_data[_index])) * int(w)) >> 10; } template class LogisticAdaptiveProbMap { public: LogisticAdaptiveProbMap(int n); ~LogisticAdaptiveProbMap() { delete[] _data; } int get(int bit, int pr, int ctx); private: int _index; // last p, context uint16* _data; // [NbCtx][33]: p, context -> p }; template inline LogisticAdaptiveProbMap::LogisticAdaptiveProbMap(int n) { const int mult = (FAST == false) ? 33 : 32; _index = 0; const int size = (n == 0) ? mult : n * mult; _data = new uint16[size]; for (int j = 0; j < mult; j++) _data[j] = uint16(Global::squash((j - 16) * 128) << 4); for (int i = 1; i < n; i++) memcpy(&_data[i * mult], &_data[0], mult * sizeof(uint16)); } // Return improved prediction given current bit, prediction and context template inline int LogisticAdaptiveProbMap::get(int bit, int pr, int ctx) { // Update probability based on error and learning rate const int g = -bit & 65528; _data[_index] += (((g - int(_data[_index])) >> RATE) + bit); if (FAST == false) { _data[_index + 1] += (((g - int(_data[_index + 1])) >> RATE) + bit); pr = Global::stretch(pr); _index = ((pr + 2048) >> 7) + 33 * ctx; // Return interpolated probabibility const uint16 w = uint16(pr & 127); return ((_data[_index] << 7) + (int(_data[_index + 1]) - int(_data[_index])) * int(w)) >> 11; } else { _index = ((Global::stretch(pr) + 2048) >> 7) + 32 * ctx; return int(_data[_index]) >> 4; } } } #endif kanzi-cpp-2.6.0/src/entropy/BinaryEntropyDecoder.cpp000066400000000000000000000117601525576310500225170ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include "BinaryEntropyDecoder.hpp" #include "../BitStreamException.hpp" #include "../Memory.hpp" #include "EntropyUtils.hpp" using namespace kanzi; using namespace std; const uint64 BinaryEntropyDecoder::TOP = 0x00FFFFFFFFFFFFFF; const uint64 BinaryEntropyDecoder::MASK_0_56 = 0x00FFFFFFFFFFFFFF; const uint64 BinaryEntropyDecoder::MASK_0_32 = 0x00000000FFFFFFFF; const int BinaryEntropyDecoder::MAX_BLOCK_SIZE = 1 << 30; const int BinaryEntropyDecoder::MAX_CHUNK_SIZE = 1 << 26; BinaryEntropyDecoder::BinaryEntropyDecoder(InputBitStream& bitstream, Predictor* predictor, bool deallocate) : _predictor(predictor) , _bitstream(bitstream) , _deallocate(deallocate) , _sba(nullptr, 0) { if (predictor == nullptr) throw invalid_argument("Invalid null predictor parameter"); _low = 0; _high = TOP; _current = 0; _payloadEnd = 0; } BinaryEntropyDecoder::~BinaryEntropyDecoder() { _dispose(); if (_sba._array != nullptr) delete[] _sba._array; if (_deallocate) delete _predictor; } void BinaryEntropyDecoder::ensureCapacity(int required) { if (required <= _sba._length) return; const int grownSize = _sba._length + max(_sba._length >> 2, 1 << 20); int newSize = max(required, max(grownSize, 1024)); kanzi::byte* buf = new kanzi::byte[newSize]; if ((_sba._array != nullptr) && (_sba._index > 0)) memcpy(buf, _sba._array, size_t(_sba._index)); delete[] _sba._array; _sba._array = buf; _sba._length = newSize; } int BinaryEntropyDecoder::decode(kanzi::byte block[], uint blkptr, uint count) { if (count >= MAX_BLOCK_SIZE) throw invalid_argument("Invalid block size parameter (max is 1<<30)"); uint startChunk = blkptr; const uint end = blkptr + count; uint length = max(count, 64u); if (length >= MAX_CHUNK_SIZE) { // If the block is big (>=64MB), split the decoding to avoid allocating // too much memory. length = (length / 8 < MAX_CHUNK_SIZE) ? count >> 3 : count >> 4; } const uint bufSize = length + (length >> 3); ensureCapacity(int(bufSize)); // Split block into chunks, read bit array from bitstream and decode chunk while (startChunk < end) { const uint chunkSize = min(length, end - startChunk); const uint szBytes = uint(EntropyUtils::readVarInt(_bitstream)); const uint64 maxEncodedSize = min(uint64(chunkSize) << 5, uint64(uint(-1) >> 3)); if (uint64(szBytes) > maxEncodedSize) return 0; ensureCapacity(int(szBytes)); _payloadEnd = szBytes; _current = _bitstream.readBits(56); if (szBytes != 0) _bitstream.readBits(&_sba._array[0], 8 * szBytes); _sba._index = 0; const uint endChunk = startChunk + chunkSize; for (uint i = startChunk; i < endChunk; i++) { block[i] = kanzi::byte((decodeBit(_predictor->get()) << 7) | (decodeBit(_predictor->get()) << 6) | (decodeBit(_predictor->get()) << 5) | (decodeBit(_predictor->get()) << 4) | (decodeBit(_predictor->get()) << 3) | (decodeBit(_predictor->get()) << 2) | (decodeBit(_predictor->get()) << 1) | decodeBit(_predictor->get())); } if (uint(_sba._index) != _payloadEnd) return 0; startChunk = endChunk; } return count; } // no inline void BinaryEntropyDecoder::read() { if (KANZI_UNLIKELY(uint(_sba._index) + 4 > _payloadEnd)) throw BitStreamException("Invalid bitstream: binary entropy payload underrun", BitStreamException::INVALID_STREAM); _low = (_low << 32) & MASK_0_56; _high = ((_high << 32) | MASK_0_32) & MASK_0_56; const uint64 val = BigEndian::readInt32(&_sba._array[_sba._index]) & MASK_0_32; _current = ((_current << 32) | val) & MASK_0_56; _sba._index += 4; } // no inline kanzi::byte BinaryEntropyDecoder::decodeByte() { return kanzi::byte((decodeBit(_predictor->get()) << 7) | (decodeBit(_predictor->get()) << 6) | (decodeBit(_predictor->get()) << 5) | (decodeBit(_predictor->get()) << 4) | (decodeBit(_predictor->get()) << 3) | (decodeBit(_predictor->get()) << 2) | (decodeBit(_predictor->get()) << 1) | decodeBit(_predictor->get())); } kanzi-cpp-2.6.0/src/entropy/BinaryEntropyDecoder.hpp000066400000000000000000000045161525576310500225250ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_BinaryEntropyDecoder #define knz_BinaryEntropyDecoder #include "../EntropyDecoder.hpp" #include "../Predictor.hpp" #include "../SliceArray.hpp" namespace kanzi { // This class is a generic implementation of a bool entropy decoder class BinaryEntropyDecoder FINAL : public EntropyDecoder { private: static const uint64 TOP; static const uint64 MASK_0_56; static const uint64 MASK_0_32; static const int MAX_BLOCK_SIZE; static const int MAX_CHUNK_SIZE; Predictor* _predictor; uint64 _low; uint64 _high; uint64 _current; InputBitStream& _bitstream; bool _deallocate; SliceArray _sba; uint _payloadEnd; void read(); void ensureCapacity(int required); void _dispose() const {} public: BinaryEntropyDecoder(InputBitStream& bitstream, Predictor* predictor, bool deallocate=true); ~BinaryEntropyDecoder(); int decode(byte block[], uint blkptr, uint count); InputBitStream& getBitStream() const { return _bitstream; } void dispose() { _dispose(); } byte decodeByte(); int decodeBit(int pred = 2048); }; inline int BinaryEntropyDecoder::decodeBit(int pred) { // Calculate interval split const uint64 split = ((((_high - _low) >> 4) * uint64(pred)) >> 8) + _low; int bit; // Update predictor if (split >= _current) { bit = 1; _high = split; _predictor->update(1); } else { bit = 0; _low = split + 1; _predictor->update(0); } // Read 32 bits from bitstream if (((_low ^ _high) >> 24) == 0) read(); return bit; } } #endif kanzi-cpp-2.6.0/src/entropy/BinaryEntropyEncoder.cpp000066400000000000000000000116071525576310500225310ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include "BinaryEntropyEncoder.hpp" #include "../Memory.hpp" #include "EntropyUtils.hpp" using namespace kanzi; using namespace std; const uint64 BinaryEntropyEncoder::TOP = 0x00FFFFFFFFFFFFFF; const uint64 BinaryEntropyEncoder::MASK_0_24 = 0x0000000000FFFFFF; const uint64 BinaryEntropyEncoder::MASK_0_32 = 0x00000000FFFFFFFF; const int BinaryEntropyEncoder::MAX_BLOCK_SIZE = 1 << 30; const int BinaryEntropyEncoder::MAX_CHUNK_SIZE = 1 << 26; BinaryEntropyEncoder::BinaryEntropyEncoder(OutputBitStream& bitstream, Predictor* predictor, bool deallocate) : _predictor(predictor) , _bitstream(bitstream) , _deallocate(deallocate) , _sba(nullptr, 0) { if (predictor == nullptr) throw invalid_argument("Invalid null predictor parameter"); _low = 0; _high = TOP; _disposed = false; } BinaryEntropyEncoder::~BinaryEntropyEncoder() { // Destructors must not replace an encoding error or terminate the process // while unwinding after a bitstream failure. try { _dispose(); } catch (...) { } if (_sba._array != nullptr) delete[] _sba._array; if (_deallocate) delete _predictor; } void BinaryEntropyEncoder::ensureCapacity(int required) { if (required <= _sba._length) return; const int grownSize = _sba._length + max(_sba._length >> 2, 1 << 10); int newSize = max(required, max(grownSize, 1024)); kanzi::byte* buf = new kanzi::byte[newSize]; if ((_sba._array != nullptr) && (_sba._index > 0)) memcpy(buf, _sba._array, size_t(_sba._index)); delete[] _sba._array; _sba._array = buf; _sba._length = newSize; } int BinaryEntropyEncoder::encode(const kanzi::byte block[], uint blkptr, uint count) { if (count >= MAX_BLOCK_SIZE) throw invalid_argument("Invalid block size parameter (max is 1<<30)"); uint startChunk = blkptr; const uint end = blkptr + count; uint length = max(count, 64u); if (length >= MAX_CHUNK_SIZE) { // If the block is big (>=64MB), split the encoding to avoid allocating // too much memory. length = (length / 8 < MAX_CHUNK_SIZE) ? count >> 3 : count >> 4; } const uint extra = max(length >> 3, min(length, uint(1 << 16))); const uint64 bufSize = uint64(length) + uint64(extra); ensureCapacity(int(min(bufSize, uint64(0x7FFFFFFF)))); // Split block into chunks, encode chunk and write bit array to bitstream while (startChunk < end) { const uint chunkSize = min(length, end - startChunk); const uint endChunk = startChunk + chunkSize; _sba._index = 0; for (uint i = startChunk; i < endChunk; i++) { encodeBit(int(block[i]) & 0x80, _predictor->get()); encodeBit(int(block[i]) & 0x40, _predictor->get()); encodeBit(int(block[i]) & 0x20, _predictor->get()); encodeBit(int(block[i]) & 0x10, _predictor->get()); encodeBit(int(block[i]) & 0x08, _predictor->get()); encodeBit(int(block[i]) & 0x04, _predictor->get()); encodeBit(int(block[i]) & 0x02, _predictor->get()); encodeBit(int(block[i]) & 0x01, _predictor->get()); } EntropyUtils::writeVarInt(_bitstream, uint32(_sba._index)); _bitstream.writeBits(&_sba._array[0], 8 * _sba._index); startChunk = endChunk; if (startChunk < end) _bitstream.writeBits(_low | MASK_0_24, 56); } return count; } void BinaryEntropyEncoder::_dispose() { if (_disposed == true) return; _disposed = true; _bitstream.writeBits(_low | MASK_0_24, 56); } void BinaryEntropyEncoder::flush() { if (_sba._length - _sba._index < 4) ensureCapacity(_sba._index + 4); BigEndian::writeInt32(&_sba._array[_sba._index], int32(_high >> 24)); _sba._index += 4; _low <<= 32; _high = (_high << 32) | MASK_0_32; } void BinaryEntropyEncoder::encodeByte(kanzi::byte val) { encodeBit(int(val) & 0x80, _predictor->get()); encodeBit(int(val) & 0x40, _predictor->get()); encodeBit(int(val) & 0x20, _predictor->get()); encodeBit(int(val) & 0x10, _predictor->get()); encodeBit(int(val) & 0x08, _predictor->get()); encodeBit(int(val) & 0x04, _predictor->get()); encodeBit(int(val) & 0x02, _predictor->get()); encodeBit(int(val) & 0x01, _predictor->get()); } kanzi-cpp-2.6.0/src/entropy/BinaryEntropyEncoder.hpp000066400000000000000000000042571525576310500225410ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_BinaryEntropyEncoder #define knz_BinaryEntropyEncoder #include "../EntropyEncoder.hpp" #include "../Predictor.hpp" #include "../SliceArray.hpp" namespace kanzi { // This class is a generic implementation of a bool entropy encoder class BinaryEntropyEncoder FINAL : public EntropyEncoder { private: static const uint64 TOP; static const uint64 MASK_0_24; static const uint64 MASK_0_32; static const int MAX_BLOCK_SIZE; static const int MAX_CHUNK_SIZE; Predictor* _predictor; uint64 _low; uint64 _high; OutputBitStream& _bitstream; bool _disposed; bool _deallocate; SliceArray _sba; void _dispose(); void ensureCapacity(int required); void flush(); public: BinaryEntropyEncoder(OutputBitStream& bitstream, Predictor* predictor, bool deallocate=true); ~BinaryEntropyEncoder(); int encode(const byte block[], uint blkptr, uint count); OutputBitStream& getBitStream() const { return _bitstream; } void dispose() { _dispose(); } void encodeByte(byte val); void encodeBit(int bit, int pred = 2048); }; inline void BinaryEntropyEncoder::encodeBit(int bit, int pred) { // Update fields with new interval bounds and predictor const uint64 mid = _low + ((((_high - _low) >> 4) * uint64(pred)) >> 8); (bit != 0) ? _high = mid : _low = mid + 1; _predictor->update(bit != 0); // Write unchanged first 32 bits to bitstream if (((_low ^ _high) >> 24) == 0) flush(); } } #endif kanzi-cpp-2.6.0/src/entropy/CMPredictor.cpp000066400000000000000000000027061525576310500205770ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include "CMPredictor.hpp" using namespace kanzi; const int CMPredictor::FAST_RATE = 2; const int CMPredictor::MEDIUM_RATE = 4; const int CMPredictor::SLOW_RATE = 6; const int CMPredictor::PSCALE = 65536; CMPredictor::CMPredictor(Context* pCtx) { _ctx = 1; _runMask = 0; _c1 = 0; _c2 = 0; const int bsVersion = (pCtx == nullptr) ? 7 : pCtx->getInt("bsVersion", 7); for (int i = 0; i < 256; i++) { for (int j = 0; j <= 256; j++) _counter1[i][j] = 32768; for (int j = 0; j <= 16; j++) { _counter2[2 * i][j] = j << 12; _counter2[2 * i + 1][j] = j << 12; } // Keep backward compatibility with old streams if (bsVersion >= 7) { _counter2[2 * i][16] = 65535; _counter2[2 * i + 1][16] = 65535; } } _pc1 = _counter1[_ctx]; _pc2 = &_counter2[_ctx][8]; } kanzi-cpp-2.6.0/src/entropy/CMPredictor.hpp000066400000000000000000000047001525576310500206000ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_CMPredictor #define knz_CMPredictor #include "../Context.hpp" #include "../Predictor.hpp" #include "../types.hpp" namespace kanzi { class CMPredictor FINAL : public Predictor { private: static const int FAST_RATE; static const int MEDIUM_RATE; static const int SLOW_RATE; static const int PSCALE; int _c1; int _c2; int _ctx; int _runMask; int _counter1[256][257]; int _counter2[512][17]; int* _pc1; int* _pc2; public: CMPredictor(Context* pCtx = nullptr); ~CMPredictor(){} void update(int bit); int get(); }; // Update the probability model inline void CMPredictor::update(int bit) { if (bit == 0) { _pc1[256] -= (_pc1[256] >> FAST_RATE); _pc1[_c1] -= (_pc1[_c1] >> MEDIUM_RATE); _pc2[0] -= (_pc2[0]>> SLOW_RATE); _pc2[1] -= (_pc2[1]>> SLOW_RATE); _ctx += _ctx; } else { _pc1[256] -= ((_pc1[256] - PSCALE + 16) >> FAST_RATE); _pc1[_c1] -= ((_pc1[_c1] - PSCALE + 16) >> MEDIUM_RATE); _pc2[0] -= ((_pc2[0] - PSCALE + 16) >> SLOW_RATE); _pc2[1] -= ((_pc2[1] - PSCALE + 16) >> SLOW_RATE); _ctx += (_ctx + 1); } if (_ctx > 255) { _c2 = _c1; _c1 = _ctx & 0xFF; _ctx = 1; _runMask = (_c1 == _c2) ? 0x100 : 0; _pc1 = _counter1[1]; } else { _pc1 = _counter1[_ctx]; } } // Return the split value representing the probability of 1 in the [0..4095] range. inline int CMPredictor::get() { const int p = (13 * (_pc1[256] + _pc1[_c1]) + 6 * _pc1[_c2]) >> 5; _pc2 = &_counter2[_ctx | _runMask][p >> 12]; return (p + p + 3 * (_pc2[0] + _pc2[1]) + 64) >> 7; // rescale to [0..4095] } } #endif kanzi-cpp-2.6.0/src/entropy/EntropyDecoderFactory.hpp000066400000000000000000000114511525576310500227040ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_EntropyDecoderFactory #define knz_EntropyDecoderFactory #include #include "../util/strings.hpp" #include "../Context.hpp" #include "ANSRangeDecoder.hpp" #include "BinaryEntropyDecoder.hpp" #include "HuffmanDecoder.hpp" #include "NullEntropyDecoder.hpp" #include "RangeDecoder.hpp" #include "CMPredictor.hpp" #include "FPAQDecoder.hpp" #include "TPAQPredictor.hpp" namespace kanzi { class EntropyDecoderFactory { public: static const short NONE_TYPE = 0; // No compression static const short HUFFMAN_TYPE = 1; // Huffman static const short FPAQ_TYPE = 2; // Fast PAQ (order 0) static const short PAQ_TYPE = 3; // Obsolete static const short RANGE_TYPE = 4; // Range static const short ANS0_TYPE = 5; // Asymmetric Numerical System order 0 static const short CM_TYPE = 6; // Context Model static const short TPAQ_TYPE = 7; // Tangelo PAQ static const short ANS1_TYPE = 8; // Asymmetric Numerical System order 1 static const short TPAQX_TYPE = 9; // Tangelo PAQ Extra static const short RESERVED1 = 10; //Reserved static const short RESERVED2 = 11; //Reserved static const short RESERVED3 = 12; //Reserved static const short RESERVED4 = 13; //Reserved static const short RESERVED5 = 14; //Reserved static const short RESERVED6 = 15; //Reserved static EntropyDecoder* newDecoder(InputBitStream& ibs, Context& ctx, short entropyType); static const char* getName(short entropyType); static short getType(const char* name); }; inline EntropyDecoder* EntropyDecoderFactory::newDecoder(InputBitStream& ibs, Context& ctx, short entropyType) { switch (entropyType) { // Each block is decoded separately // Rebuild the entropy decoder to reset block statistics case HUFFMAN_TYPE: return new HuffmanDecoder(ibs, &ctx); case ANS0_TYPE: return new ANSRangeDecoder(ibs, 0); case ANS1_TYPE: return new ANSRangeDecoder(ibs, 1); case RANGE_TYPE: return new RangeDecoder(ibs); case FPAQ_TYPE: return new FPAQDecoder(ibs); case CM_TYPE: return new BinaryEntropyDecoder(ibs, new CMPredictor(&ctx)); case TPAQ_TYPE: return new BinaryEntropyDecoder(ibs, new TPAQPredictor(&ctx)); case TPAQX_TYPE: return new BinaryEntropyDecoder(ibs, new TPAQPredictor(&ctx)); case NONE_TYPE: return new NullEntropyDecoder(ibs); default: std::string msg = "Unknown entropy codec type: '"; msg += char(entropyType); msg += '\''; throw std::invalid_argument(msg); } } inline const char* EntropyDecoderFactory::getName(short entropyType) { switch (entropyType) { case HUFFMAN_TYPE: return "HUFFMAN"; case ANS0_TYPE: return "ANS0"; case ANS1_TYPE: return "ANS1"; case RANGE_TYPE: return "RANGE"; case FPAQ_TYPE: return "FPAQ"; case CM_TYPE: return "CM"; case TPAQ_TYPE: return "TPAQ"; case TPAQX_TYPE: return "TPAQX"; case NONE_TYPE: return "NONE"; default: std::string msg = "Unknown entropy codec type: '"; msg += char(entropyType); msg += '\''; throw std::invalid_argument(msg); } } inline short EntropyDecoderFactory::getType(const char* str) { std::string name = str; transform(name.begin(), name.end(), name.begin(), safeToUpper); if (name == "HUFFMAN") return HUFFMAN_TYPE; if (name == "ANS0") return ANS0_TYPE; if (name == "ANS1") return ANS1_TYPE; if (name == "FPAQ") return FPAQ_TYPE; if (name == "RANGE") return RANGE_TYPE; if (name == "CM") return CM_TYPE; if (name == "TPAQ") return TPAQ_TYPE; if (name == "TPAQX") return TPAQX_TYPE; if (name == "NONE") return NONE_TYPE; throw std::invalid_argument("Unsupported entropy codec type: '" + name + "'"); } } #endif kanzi-cpp-2.6.0/src/entropy/EntropyEncoderFactory.hpp000066400000000000000000000112721525576310500227170ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_EntropyEncoderFactory #define knz_EntropyEncoderFactory #include #include "../util/strings.hpp" #include "../Context.hpp" #include "ANSRangeEncoder.hpp" #include "BinaryEntropyEncoder.hpp" #include "HuffmanEncoder.hpp" #include "NullEntropyEncoder.hpp" #include "RangeEncoder.hpp" #include "CMPredictor.hpp" #include "FPAQEncoder.hpp" #include "TPAQPredictor.hpp" namespace kanzi { class EntropyEncoderFactory { public: static const short NONE_TYPE = 0; // No compression static const short HUFFMAN_TYPE = 1; // Huffman static const short FPAQ_TYPE = 2; // Fast PAQ (order 0) static const short PAQ_TYPE = 3; // Obsolete static const short RANGE_TYPE = 4; // Range static const short ANS0_TYPE = 5; // Asymmetric Numerical System order 0 static const short CM_TYPE = 6; // Context Model static const short TPAQ_TYPE = 7; // Tangelo PAQ static const short ANS1_TYPE = 8; // Asymmetric Numerical System order 1 static const short TPAQX_TYPE = 9; // Tangelo PAQ Extra static const short RESERVED1 = 10; //Reserved static const short RESERVED2 = 11; //Reserved static const short RESERVED3 = 12; //Reserved static const short RESERVED4 = 13; //Reserved static const short RESERVED5 = 14; //Reserved static const short RESERVED6 = 15; //Reserved static EntropyEncoder* newEncoder(OutputBitStream& obs, Context& ctx, short entropyType); static const char* getName(short entropyType); static short getType(const char* name); }; inline EntropyEncoder* EntropyEncoderFactory::newEncoder(OutputBitStream& obs, Context& ctx, short entropyType) { switch (entropyType) { case HUFFMAN_TYPE: return new HuffmanEncoder(obs); case ANS0_TYPE: return new ANSRangeEncoder(obs, 0); case ANS1_TYPE: return new ANSRangeEncoder(obs, 1); case RANGE_TYPE: return new RangeEncoder(obs); case FPAQ_TYPE: return new FPAQEncoder(obs); case CM_TYPE: return new BinaryEntropyEncoder(obs, new CMPredictor(&ctx)); case TPAQ_TYPE: return new BinaryEntropyEncoder(obs, new TPAQPredictor(&ctx)); case TPAQX_TYPE: return new BinaryEntropyEncoder(obs, new TPAQPredictor(&ctx)); case NONE_TYPE: return new NullEntropyEncoder(obs); default: std::string msg = "Unknown entropy codec type: '"; msg += char(entropyType); msg += '\''; throw std::invalid_argument(msg); } } inline const char* EntropyEncoderFactory::getName(short entropyType) { switch (entropyType) { case HUFFMAN_TYPE: return "HUFFMAN"; case ANS0_TYPE: return "ANS0"; case ANS1_TYPE: return "ANS1"; case RANGE_TYPE: return "RANGE"; case FPAQ_TYPE: return "FPAQ"; case CM_TYPE: return "CM"; case TPAQ_TYPE: return "TPAQ"; case TPAQX_TYPE: return "TPAQX"; case NONE_TYPE: return "NONE"; default: std::string msg = "Unknown entropy codec type: '"; msg += char(entropyType); msg += '\''; throw std::invalid_argument(msg); } } inline short EntropyEncoderFactory::getType(const char* str) { std::string name = str; transform(name.begin(), name.end(), name.begin(), safeToUpper); if (name == "HUFFMAN") return HUFFMAN_TYPE; if (name == "ANS0") return ANS0_TYPE; if (name == "ANS1") return ANS1_TYPE; if (name == "FPAQ") return FPAQ_TYPE; if (name == "RANGE") return RANGE_TYPE; if (name == "CM") return CM_TYPE; if (name == "TPAQ") return TPAQ_TYPE; if (name == "TPAQX") return TPAQX_TYPE; if (name == "NONE") return NONE_TYPE; throw std::invalid_argument("Unsupported entropy codec type: '" + name + "'"); } } #endif kanzi-cpp-2.6.0/src/entropy/EntropyUtils.cpp000066400000000000000000000165151525576310500211100ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include "EntropyUtils.hpp" #include "../BitStreamException.hpp" using namespace kanzi; using namespace std; const int EntropyUtils::FULL_ALPHABET = 0; const int EntropyUtils::PARTIAL_ALPHABET = 1; const int EntropyUtils::ALPHABET_256 = 0; const int EntropyUtils::ALPHABET_0 = 1; const int EntropyUtils::INCOMPRESSIBLE_THRESHOLD = 973; // 0.95*1024 class FreqSortData { public: uint* _freq; uint8 _symbol; FreqSortData(uint* freq, uint8 symbol) : _freq(freq) , _symbol(symbol) { } }; struct FreqDataComparator { bool operator()(FreqSortData const& fd1, FreqSortData const& fd2) const { // Decreasing frequency then decreasing symbol int r; return ((r = int(*fd1._freq - *fd2._freq)) == 0) ? fd1._symbol > fd2._symbol: r > 0; } }; // alphabet must be sorted in increasing order // length = alphabet array length up to 256 int EntropyUtils::encodeAlphabet(OutputBitStream& obs, const uint alphabet[], int length, int count) { // Alphabet length must be a power of 2 if ((length & (length - 1)) != 0) return -1; if ((length > 256) || (count > length)) return -1; if (count == 0) { obs.writeBit(FULL_ALPHABET); obs.writeBit(ALPHABET_0); } else if (count == 256) { obs.writeBit(FULL_ALPHABET); obs.writeBit(ALPHABET_256); } else { // Partial alphabet obs.writeBit(PARTIAL_ALPHABET); kanzi::byte masks[32] = { kanzi::byte(0) }; // Encode presence flags for (int i = 0; i < count; i++) masks[alphabet[i] >> 3] |= kanzi::byte(1 << (alphabet[i] & 7)); const int lastMask = alphabet[count - 1] >> 3; obs.writeBits(lastMask, 5); obs.writeBits(masks, 8 * (lastMask + 1)); } return count; } int EntropyUtils::decodeAlphabet(InputBitStream& ibs, uint alphabet[]) { // Read encoding mode from bitstream if (ibs.readBit() == FULL_ALPHABET) { const int alphabetSize = (ibs.readBit() == ALPHABET_256) ? 256 : 0; // Full alphabet for (int i = 0; i < alphabetSize; i++) alphabet[i] = i; return alphabetSize; } // Partial alphabet const int lastMask = int(ibs.readBits(5)); kanzi::byte masks[32] = { kanzi::byte(0) }; int count = 0; // Decode presence flags ibs.readBits(masks, 8 * (lastMask + 1)); for (int i = 0; i <= lastMask; i++) { const int n = 8 * i; for (uint j = 0; j < 8; j++) { const int bit = int(masks[i] >> j) & 1; alphabet[count] = n + j; count += bit; } } return count; } // Returns the size of the alphabet // length is the length of the alphabet array // 'totalFreq' is the sum of frequencies. // 'scale' is the target new total of frequencies // The alphabet and freqs parameters are updated int EntropyUtils::normalizeFrequencies(uint freqs[], uint alphabet[], int length, uint totalFreq, uint scale) { if (length > 256) { stringstream ss; ss << "Invalid alphabet size parameter: " << scale << " (must be less than or equal to 256)"; throw invalid_argument(ss.str()); } if ((scale < 256) || (scale > 65536)) { stringstream ss; ss << "Invalid scale parameter: " << scale << " (must be in [256..65536])"; throw invalid_argument(ss.str()); } if ((length == 0) || (totalFreq == 0)) return 0; // Number of present symbols int alphabetSize = 0; // shortcut if (totalFreq == scale) { for (int i = 0; i < length; i++) { if (freqs[i] != 0) alphabet[alphabetSize++] = i; } return alphabetSize; } uint sumScaledFreq = 0; uint sumFreq = 0; int idxMax = 0; // Scale frequencies by squeezing/stretching distribution over complete range for (int i = 0; i < length; i++) { alphabet[i] = 0; const uint f = freqs[i]; if (f == 0) continue; alphabet[alphabetSize++] = i; const int64 sf = int64(f) * int64(scale); const uint scaledFreq = sf <= int64(totalFreq) ? 1 : uint((sf + (int64(totalFreq) >> 1)) / int64(totalFreq)); sumScaledFreq += scaledFreq; freqs[i] = scaledFreq; sumFreq += f; idxMax = (scaledFreq > freqs[idxMax]) ? i : idxMax; if (sumFreq >= totalFreq) break; } if (alphabetSize == 0) return 0; if (alphabetSize == 1) { freqs[alphabet[0]] = scale; return 1; } if (sumScaledFreq == scale) return alphabetSize; int delta = int(sumScaledFreq - scale); const int errThr = int(freqs[idxMax]) >> 4; if (abs(delta) <= errThr) { // Fast path (small error): just adjust the max frequency freqs[idxMax] -= delta; return alphabetSize; } if (delta < 0) { delta += errThr; freqs[idxMax] += uint(errThr); } else { delta -= errThr; freqs[idxMax] -= uint(errThr); } // Slow path: spread error across frequencies const int inc = (delta < 0) ? 1 : -1; delta = abs(delta); int round = 0; while ((++round < 6) && (delta > 0)) { int adjustments = 0; for (int i = 0; i < alphabetSize; i++) { const int idx = alphabet[i]; // Skip small frequencies to avoid big distortion // Do not zero out frequencies if (freqs[idx] <= 2) continue; // Adjust frequency freqs[idx] += inc; adjustments++; delta--; if (delta == 0) break; } if (adjustments == 0) break; } freqs[idxMax] = max(freqs[idxMax] - delta, uint(1)); return alphabetSize; } int EntropyUtils::writeVarInt(OutputBitStream& obs, uint32 value) { uint32 res = 0; while (value >= 128) { obs.writeBits(0x80 | (value & 0x7F), 8); value >>= 7; res++; } obs.writeBits(value, 8); return res; } uint32 EntropyUtils::readVarInt(InputBitStream& ibs) { uint32 value = uint32(ibs.readBits(8)); uint32 res = value & 0x7F; for (int shift = 7; value >= 128; shift += 7) { value = uint32(ibs.readBits(8)); if (shift == 28) { // uint32 varint: last byte may only carry 4 payload bits and // must terminate the sequence. if ((value >= 128) || ((value & 0x70) != 0)) { throw BitStreamException("Invalid variable-length integer in bitstream", BitStreamException::INVALID_STREAM); } res |= ((value & 0x0F) << shift); return res; } res |= ((value & 0x7F) << shift); } return res; } kanzi-cpp-2.6.0/src/entropy/EntropyUtils.hpp000066400000000000000000000026721525576310500211140ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_EntropyUtils #define knz_EntropyUtils #include "../InputBitStream.hpp" #include "../OutputBitStream.hpp" namespace kanzi { class EntropyUtils { private: static const int FULL_ALPHABET; static const int PARTIAL_ALPHABET; static const int ALPHABET_256; static const int ALPHABET_0; public: static const int INCOMPRESSIBLE_THRESHOLD; EntropyUtils() {} ~EntropyUtils() {} static int encodeAlphabet(OutputBitStream& obs, const uint alphabet[], int length, int count); static int decodeAlphabet(InputBitStream& ibs, uint alphabet[]); static int normalizeFrequencies(uint freqs[], uint alphabet[], int length, uint totalFreq, uint scale); static int writeVarInt(OutputBitStream& obs, uint32 val); static uint32 readVarInt(InputBitStream& ibs); }; } #endif kanzi-cpp-2.6.0/src/entropy/ExpGolombDecoder.cpp000066400000000000000000000024361525576310500216060ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include "ExpGolombDecoder.hpp" using namespace kanzi; ExpGolombDecoder::ExpGolombDecoder(InputBitStream& bitstream, bool sgn) : _bitstream(bitstream), _signed(sgn) { } int ExpGolombDecoder::decode(kanzi::byte block[], uint blkptr, uint len) { kanzi::byte* buf = &block[blkptr]; const uint len8 = len & uint(-8); for (uint i = 0; i < len8; i += 8) { buf[i] = decodeByte(); buf[i+1] = decodeByte(); buf[i+2] = decodeByte(); buf[i+3] = decodeByte(); buf[i+4] = decodeByte(); buf[i+5] = decodeByte(); buf[i+6] = decodeByte(); buf[i+7] = decodeByte(); } for (uint i = len8; i < len; i++) buf[i] = decodeByte(); return len; } kanzi-cpp-2.6.0/src/entropy/ExpGolombDecoder.hpp000066400000000000000000000041301525576310500216040ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_ExpGolombDecoder #define knz_ExpGolombDecoder #include "../EntropyDecoder.hpp" namespace kanzi { class ExpGolombDecoder : public EntropyDecoder { private: InputBitStream& _bitstream; const bool _signed; void flush(); void _dispose() const {} public: ExpGolombDecoder(InputBitStream& bitstream, bool sign=true); ~ExpGolombDecoder() { _dispose(); } int decode(byte arr[], uint blkptr, uint len); InputBitStream& getBitStream() const { return _bitstream; } byte decodeByte(); void dispose() { _dispose(); } bool isSigned() const { return _signed; } }; inline byte ExpGolombDecoder::decodeByte() { if (_bitstream.readBit() != 0) return byte(0); uint log2 = 1; while (_bitstream.readBit() == 0) log2++; // Clamp. Do not attempt to detect a corrupted bitstream. // Unsigned byte 255 requires an 8-bit suffix; signed values require at most 7. log2 = log2 <= 8u - uint(_signed) ? log2 : 8u - uint(_signed); const uint base = (1u << log2) - 1; if (_signed == true) { // Decode signed: read value + sign const uint res = uint(_bitstream.readBits(log2 + 1)); const uint value = (res >> 1) + base; const uint sgn = res & 1; return byte((value ^ (0u - sgn)) + sgn); // value or -value } // Decode unsigned return byte(base + uint(_bitstream.readBits(log2))); } } #endif kanzi-cpp-2.6.0/src/entropy/ExpGolombEncoder.cpp000066400000000000000000000112701525576310500216140ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include "ExpGolombEncoder.hpp" using namespace kanzi; const int ExpGolombEncoder::CACHE[2][256] = { // Unsigned { 513, 1538, 1539, 2564, 2565, 2566, 2567, 3592, 3593, 3594, 3595, 3596, 3597, 3598, 3599, 4624, 4625, 4626, 4627, 4628, 4629, 4630, 4631, 4632, 4633, 4634, 4635, 4636, 4637, 4638, 4639, 5664, 5665, 5666, 5667, 5668, 5669, 5670, 5671, 5672, 5673, 5674, 5675, 5676, 5677, 5678, 5679, 5680, 5681, 5682, 5683, 5684, 5685, 5686, 5687, 5688, 5689, 5690, 5691, 5692, 5693, 5694, 5695, 6720, 6721, 6722, 6723, 6724, 6725, 6726, 6727, 6728, 6729, 6730, 6731, 6732, 6733, 6734, 6735, 6736, 6737, 6738, 6739, 6740, 6741, 6742, 6743, 6744, 6745, 6746, 6747, 6748, 6749, 6750, 6751, 6752, 6753, 6754, 6755, 6756, 6757, 6758, 6759, 6760, 6761, 6762, 6763, 6764, 6765, 6766, 6767, 6768, 6769, 6770, 6771, 6772, 6773, 6774, 6775, 6776, 6777, 6778, 6779, 6780, 6781, 6782, 6783, 7808, 7809, 7810, 7811, 7812, 7813, 7814, 7815, 7816, 7817, 7818, 7819, 7820, 7821, 7822, 7823, 7824, 7825, 7826, 7827, 7828, 7829, 7830, 7831, 7832, 7833, 7834, 7835, 7836, 7837, 7838, 7839, 7840, 7841, 7842, 7843, 7844, 7845, 7846, 7847, 7848, 7849, 7850, 7851, 7852, 7853, 7854, 7855, 7856, 7857, 7858, 7859, 7860, 7861, 7862, 7863, 7864, 7865, 7866, 7867, 7868, 7869, 7870, 7871, 7872, 7873, 7874, 7875, 7876, 7877, 7878, 7879, 7880, 7881, 7882, 7883, 7884, 7885, 7886, 7887, 7888, 7889, 7890, 7891, 7892, 7893, 7894, 7895, 7896, 7897, 7898, 7899, 7900, 7901, 7902, 7903, 7904, 7905, 7906, 7907, 7908, 7909, 7910, 7911, 7912, 7913, 7914, 7915, 7916, 7917, 7918, 7919, 7920, 7921, 7922, 7923, 7924, 7925, 7926, 7927, 7928, 7929, 7930, 7931, 7932, 7933, 7934, 7935, 8960 }, // Signed { 513, 2052, 2054, 3080, 3082, 3084, 3086, 4112, 4114, 4116, 4118, 4120, 4122, 4124, 4126, 5152, 5154, 5156, 5158, 5160, 5162, 5164, 5166, 5168, 5170, 5172, 5174, 5176, 5178, 5180, 5182, 6208, 6210, 6212, 6214, 6216, 6218, 6220, 6222, 6224, 6226, 6228, 6230, 6232, 6234, 6236, 6238, 6240, 6242, 6244, 6246, 6248, 6250, 6252, 6254, 6256, 6258, 6260, 6262, 6264, 6266, 6268, 6270, 7296, 7298, 7300, 7302, 7304, 7306, 7308, 7310, 7312, 7314, 7316, 7318, 7320, 7322, 7324, 7326, 7328, 7330, 7332, 7334, 7336, 7338, 7340, 7342, 7344, 7346, 7348, 7350, 7352, 7354, 7356, 7358, 7360, 7362, 7364, 7366, 7368, 7370, 7372, 7374, 7376, 7378, 7380, 7382, 7384, 7386, 7388, 7390, 7392, 7394, 7396, 7398, 7400, 7402, 7404, 7406, 7408, 7410, 7412, 7414, 7416, 7418, 7420, 7422, 8448, 8451, 8449, 7423, 7421, 7419, 7417, 7415, 7413, 7411, 7409, 7407, 7405, 7403, 7401, 7399, 7397, 7395, 7393, 7391, 7389, 7387, 7385, 7383, 7381, 7379, 7377, 7375, 7373, 7371, 7369, 7367, 7365, 7363, 7361, 7359, 7357, 7355, 7353, 7351, 7349, 7347, 7345, 7343, 7341, 7339, 7337, 7335, 7333, 7331, 7329, 7327, 7325, 7323, 7321, 7319, 7317, 7315, 7313, 7311, 7309, 7307, 7305, 7303, 7301, 7299, 7297, 6271, 6269, 6267, 6265, 6263, 6261, 6259, 6257, 6255, 6253, 6251, 6249, 6247, 6245, 6243, 6241, 6239, 6237, 6235, 6233, 6231, 6229, 6227, 6225, 6223, 6221, 6219, 6217, 6215, 6213, 6211, 6209, 5183, 5181, 5179, 5177, 5175, 5173, 5171, 5169, 5167, 5165, 5163, 5161, 5159, 5157, 5155, 5153, 4127, 4125, 4123, 4121, 4119, 4117, 4115, 4113, 3087, 3085, 3083, 3081, 2055, 2053 } }; ExpGolombEncoder::ExpGolombEncoder(OutputBitStream& bitstream, bool sgn) : _bitstream(bitstream), _signed((sgn == true) ? 1 : 0) { } int ExpGolombEncoder::encode(const kanzi::byte block[], uint blkptr, uint len) { const kanzi::byte* buf = &block[blkptr]; const uint len8 = len & uint(-8); for (uint i = 0; i < len8; i += 8) { encodeByte(buf[i]); encodeByte(buf[i+1]); encodeByte(buf[i+2]); encodeByte(buf[i+3]); encodeByte(buf[i+4]); encodeByte(buf[i+5]); encodeByte(buf[i+6]); encodeByte(buf[i+7]); } for (uint i = len8; i < len; i++) encodeByte(buf[i]); return len; } kanzi-cpp-2.6.0/src/entropy/ExpGolombEncoder.hpp000066400000000000000000000031141525576310500216170ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_ExpGolombEncoder #define knz_ExpGolombEncoder #include "../EntropyEncoder.hpp" namespace kanzi { class ExpGolombEncoder : public EntropyEncoder { private: static const int CACHE[2][256]; OutputBitStream& _bitstream; const int _signed; void _dispose() const {} public: ExpGolombEncoder(OutputBitStream& bitstream, bool sign=true); ~ExpGolombEncoder() { _dispose(); } int encode(const byte block[], uint blkptr, uint len); OutputBitStream& getBitStream() const { return _bitstream; } void encodeByte(byte val); void dispose() { _dispose(); } bool isSigned() const { return _signed == 1; } }; inline void ExpGolombEncoder::encodeByte(byte val) { if (val == byte(0)) { // shortcut when input is 0 _bitstream.writeBit(1); return; } const int emit = CACHE[_signed][uint8(val)]; _bitstream.writeBits(emit & 0x1FF, emit >> 9); } } #endif kanzi-cpp-2.6.0/src/entropy/FPAQDecoder.cpp000066400000000000000000000063751525576310500204470ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include "FPAQDecoder.hpp" #include "EntropyUtils.hpp" using namespace kanzi; using namespace std; const uint64 FPAQDecoder::TOP = 0x00FFFFFFFFFFFFFF; const uint64 FPAQDecoder::MASK_0_56 = 0x00FFFFFFFFFFFFFF; const uint64 FPAQDecoder::MASK_0_32 = 0x00000000FFFFFFFF; const uint FPAQDecoder::DEFAULT_CHUNK_SIZE = 4 * 1024 * 1024; const uint FPAQDecoder::MAX_BLOCK_SIZE = 1 << 30; const int FPAQDecoder::PSCALE = 65536; FPAQDecoder::FPAQDecoder(InputBitStream& bitstream) : _bitstream(bitstream) { reset(); } FPAQDecoder::~FPAQDecoder() { _dispose(); } bool FPAQDecoder::reset() { _low = 0; _high = TOP; _current = 0; _ctx = 1; _bufLimit = 0; _index = 0; for (int i = 0; i < 4; i++) { for (int j = 0; j < 256; j++) _probs[i][j] = PSCALE >> 1; } _p = _probs[0]; return true; } int FPAQDecoder::decode(kanzi::byte block[], uint blkptr, uint count) { if (count >= MAX_BLOCK_SIZE) throw invalid_argument("Invalid block size parameter (max is 1<<30)"); uint startChunk = blkptr; const uint end = blkptr + count; // Read bit array from bitstream and decode chunk while (startChunk < end) { const uint chunkSize = min(DEFAULT_CHUNK_SIZE, end - startChunk); const uint szBytes = uint(EntropyUtils::readVarInt(_bitstream)); // A payload belongs to one chunk, not to the whole block. Besides // preventing excessive allocations, this keeps the bit count below // the range where '8 * szBytes' could overflow. if (uint64(szBytes) > (uint64(chunkSize) << 1)) return 0; if (_buf.size() < szBytes) _buf.resize(szBytes); _current = _bitstream.readBits(56); // read() checks _bufLimit before accessing the buffer, so no padding // or clearing beyond the payload is necessary. if (szBytes != 0) _bitstream.readBits(&_buf[0], szBytes << 3); _bufLimit = szBytes; _index = 0; const uint endChunk = startChunk + chunkSize; _p = _probs[0]; for (uint i = startChunk; i < endChunk; i++) { _ctx = 1; decodeBit(_p[_ctx]); decodeBit(_p[_ctx]); decodeBit(_p[_ctx]); decodeBit(_p[_ctx]); decodeBit(_p[_ctx]); decodeBit(_p[_ctx]); decodeBit(_p[_ctx]); decodeBit(_p[_ctx]); block[i] = kanzi::byte(_ctx); if (_index > szBytes) return 0; _p = _probs[(_ctx & 0xFF) >> 6]; } if (_index > szBytes) return 0; startChunk = endChunk; } return count; } kanzi-cpp-2.6.0/src/entropy/FPAQDecoder.hpp000066400000000000000000000060261525576310500204450ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_FPAQDecoder #define knz_FPAQDecoder #include #include "../EntropyDecoder.hpp" #include "../Memory.hpp" #include "../SliceArray.hpp" namespace kanzi { // Derived from fpaq0r by Matt Mahoney & Alexander Ratushnyak. // See http://mattmahoney.net/dc/#fpaq0. // Simple (and fast) adaptive entropy bit coder class FPAQDecoder : public EntropyDecoder { private: static const uint64 TOP; static const uint64 MASK_0_56; static const uint64 MASK_0_32; static const uint DEFAULT_CHUNK_SIZE; static const uint MAX_BLOCK_SIZE; static const int PSCALE; uint64 _low; uint64 _high; uint64 _current; InputBitStream& _bitstream; std::vector _buf; uint _bufLimit; uint _index; uint16 _probs[4][256]; // probability of bit=1 uint16* _p; // pointer to current prob int _ctx; // previous bits void _dispose() const {} int decodeBit(int pred = 2048); bool reset(); public: FPAQDecoder(InputBitStream& bitstream); ~FPAQDecoder(); int decode(byte block[], uint blkptr, uint count); InputBitStream& getBitStream() const { return _bitstream; } void dispose() { _dispose(); } void read(); }; inline int FPAQDecoder::decodeBit(int prob) { // Calculate interval split // Written in a way to maximize accuracy of multiplication/division const uint64 split = ((((_high - _low) >> 8) * uint64(prob)) >> 8) + _low; int bit; // Update probabilities if (split >= _current) { _high = split; _p[_ctx] -= uint16((_p[_ctx] - PSCALE + 64) >> 6); _ctx += (_ctx + 1); bit = 1; } else { _low = split + 1; _p[_ctx] -= uint16(_p[_ctx] >> 6); _ctx += _ctx; bit = 0; } // Read 32 bits from bitstream if (((_low ^ _high) >> 24) == 0) read(); return bit; } inline void FPAQDecoder::read() { _low = (_low << 32) & MASK_0_56; _high = ((_high << 32) | MASK_0_32) & MASK_0_56; if (_index + 4 > _bufLimit) { _current = (_current << 32) & MASK_0_56; _index = _bufLimit + 1; return; } const uint64 val = BigEndian::readInt32(&_buf[_index]) & MASK_0_32; _current = ((_current << 32) | val) & MASK_0_56; _index += 4; } } #endif kanzi-cpp-2.6.0/src/entropy/FPAQEncoder.cpp000066400000000000000000000064461525576310500204600ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include "FPAQEncoder.hpp" #include "EntropyUtils.hpp" using namespace kanzi; using namespace std; const uint64 FPAQEncoder::TOP = 0x00FFFFFFFFFFFFFF; const uint64 FPAQEncoder::MASK_0_24 = 0x0000000000FFFFFF; const uint64 FPAQEncoder::MASK_0_32 = 0x00000000FFFFFFFF; const uint FPAQEncoder::DEFAULT_CHUNK_SIZE = 4 * 1024 * 1024; const uint FPAQEncoder::MAX_BLOCK_SIZE = 1 << 30; const int FPAQEncoder::PSCALE = 65536; FPAQEncoder::FPAQEncoder(OutputBitStream& bitstream) : _bitstream(bitstream) { reset(); } FPAQEncoder::~FPAQEncoder() { // Destructors must not replace an encoding error or terminate the process // while unwinding after a bitstream failure. try { _dispose(); } catch (...) { } } bool FPAQEncoder::reset() { _index = 0; _low = 0; _high = TOP; _disposed = false; for (int i = 0; i < 4; i++) { for (int j = 0; j < 256; j++) _probs[i][j] = PSCALE >> 1; } return true; } int FPAQEncoder::encode(const kanzi::byte block[], uint blkptr, uint count) { if (count >= MAX_BLOCK_SIZE) throw invalid_argument("Invalid block size parameter (max is 1<<30)"); uint startChunk = blkptr; const uint end = blkptr + count; const uint size = min(DEFAULT_CHUNK_SIZE, count); const uint extra = max(size >> 3, min(size, uint(1 << 16))); const uint bufSize = max(size + extra, uint(1024)); if (_buf.size() < bufSize) _buf.resize(bufSize); // Split block into chunks, encode chunk and write bit array to bitstream while (startChunk < end) { const uint chunkSize = min(DEFAULT_CHUNK_SIZE, end - startChunk); _index = 0; const uint endChunk = startChunk + chunkSize; uint16* p = _probs[0]; for (uint i = startChunk; i < endChunk; i++) { const int val = int(block[i]); const int bits = val + 256; encodeBit(val & 0x80, p[1]); encodeBit(val & 0x40, p[bits >> 7]); encodeBit(val & 0x20, p[bits >> 6]); encodeBit(val & 0x10, p[bits >> 5]); encodeBit(val & 0x08, p[bits >> 4]); encodeBit(val & 0x04, p[bits >> 3]); encodeBit(val & 0x02, p[bits >> 2]); encodeBit(val & 0x01, p[bits >> 1]); p = _probs[val >> 6]; } EntropyUtils::writeVarInt(_bitstream, uint32(_index)); _bitstream.writeBits(&_buf[0], 8 * _index); startChunk += chunkSize; if (startChunk < end) _bitstream.writeBits(_low | MASK_0_24, 56); } return count; } void FPAQEncoder::_dispose() { if (_disposed == true) return; _disposed = true; _bitstream.writeBits(_low | MASK_0_24, 56); } kanzi-cpp-2.6.0/src/entropy/FPAQEncoder.hpp000066400000000000000000000051471525576310500204620ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_FPAQEncoder #define knz_FPAQEncoder #include #include "../EntropyEncoder.hpp" #include "../Memory.hpp" #include "../SliceArray.hpp" namespace kanzi { // Derived from fpaq0r by Matt Mahoney & Alexander Ratushnyak. // See http://mattmahoney.net/dc/#fpaq0. // Simple (and fast) adaptive entropy bit coder class FPAQEncoder : public EntropyEncoder { private: static const uint64 TOP; static const uint64 MASK_0_24; static const uint64 MASK_0_32; static const uint DEFAULT_CHUNK_SIZE; static const uint MAX_BLOCK_SIZE; static const int PSCALE; uint64 _low; uint64 _high; bool _disposed; OutputBitStream& _bitstream; std::vector _buf; uint _index; uint16 _probs[4][256]; // probability of bit=1 void encodeBit(int bit, uint16& prob); bool reset(); void _dispose(); public: FPAQEncoder(OutputBitStream& bitstream); ~FPAQEncoder(); int encode(const byte block[], uint blkptr, uint count); OutputBitStream& getBitStream() const { return _bitstream; } void dispose() { _dispose(); } void flush(); }; inline void FPAQEncoder::encodeBit(int bit, uint16& prob) { // Update probabilities if (bit == 0) { _low = _low + ((((_high - _low) >> 8) * uint64(prob)) >> 8) + 1; prob -= uint16(prob >> 6); } else { _high = _low + ((((_high - _low) >> 8) * uint64(prob)) >> 8); prob -= uint16((prob - PSCALE + 64) >> 6); } // Write unchanged first 32 bits to bitstream if (((_low ^ _high) >> 24) == 0) flush(); } inline void FPAQEncoder::flush() { // Reserve space for the four-byte range coder output. if (size_t(_index) + 4 > _buf.size()) _buf.resize(size_t(_index) + 4); BigEndian::writeInt32(&_buf[_index], int32(_high >> 24)); _index += 4; _low <<= 32; _high = (_high << 32) | MASK_0_32; } } #endif kanzi-cpp-2.6.0/src/entropy/HuffmanCommon.cpp000066400000000000000000000042521525576310500211570ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include "HuffmanCommon.hpp" using namespace kanzi; const int HuffmanCommon::LOG_MAX_CHUNK_SIZE = 14; const int HuffmanCommon::MAX_CHUNK_SIZE = 1 << LOG_MAX_CHUNK_SIZE; const int HuffmanCommon::MAX_SYMBOL_SIZE = 12; // Return the number of codes generated // codes and symbols are updated int HuffmanCommon::generateCanonicalCodes(const uint16 sizes[], uint16 codes[], uint symbols[], int count) { if (count == 0) return 0; if ((count < 0) || (count > 256)) return -1; if (count > 1) { uint8 present[256] = { 0 }; uint16 offsets[MAX_SYMBOL_SIZE + 1] = { 0 }; for (int i = 0; i < count; i++) { const uint s = symbols[i]; if (s > 255) return -1; const uint16 len = sizes[s]; if ((len == 0) || (len > MAX_SYMBOL_SIZE) || (present[s] != 0)) return -1; present[s] = 1; offsets[len]++; } uint16 offset = 0; for (int len = 1; len <= MAX_SYMBOL_SIZE; len++) { const uint16 n = offsets[len]; offsets[len] = offset; offset += n; } // Scanning symbols in value order preserves the previous tie break. for (uint s = 0; s < 256; s++) { if (present[s] != 0) symbols[offsets[sizes[s]]++] = s; } } int curLen = sizes[symbols[0]]; for (int i = 0, code = 0; i < count; i++) { const int s = symbols[i]; code <<= (sizes[s] - curLen); curLen = sizes[s]; codes[s] = uint16(code); code++; } return count; } kanzi-cpp-2.6.0/src/entropy/HuffmanCommon.hpp000066400000000000000000000017221525576310500211630ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_HuffmanCommon #define knz_HuffmanCommon #include "../types.hpp" namespace kanzi { class HuffmanCommon { public: static const int LOG_MAX_CHUNK_SIZE; static const int MAX_CHUNK_SIZE; static const int MAX_SYMBOL_SIZE; static int generateCanonicalCodes(const uint16 sizes[], uint16 codes[], uint ranks[], int count); }; } #endif kanzi-cpp-2.6.0/src/entropy/HuffmanDecoder.cpp000066400000000000000000000416411525576310500212770ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include "HuffmanDecoder.hpp" #include "EntropyUtils.hpp" #include "ExpGolombDecoder.hpp" #include "../BitStreamException.hpp" #include "../Memory.hpp" using namespace kanzi; using namespace std; const int HuffmanDecoder::DECODING_BATCH_SIZE = 12; // ensures decoding table fits in L1 cache const int HuffmanDecoder::TABLE_MASK = (1 << DECODING_BATCH_SIZE) - 1; static const uint HUFFMAN_FRAGMENT_GUARD_BYTES = 8; // The chunk size indicates how many bytes are encoded (per block) before // resetting the frequency stats. HuffmanDecoder::HuffmanDecoder(InputBitStream& bitstream, Context* pCtx, int chunkSize) : _bitstream(bitstream) { if (chunkSize < 1024) throw invalid_argument("Huffman codec: The chunk size must be at least 1024"); if (chunkSize > HuffmanCommon::MAX_CHUNK_SIZE) { stringstream ss; ss << "Huffman codec: The chunk size must be at most " << HuffmanCommon::MAX_CHUNK_SIZE; throw invalid_argument(ss.str()); } _chunkSize = chunkSize; _buffer = nullptr; _bufferSize = 0; _pCtx = pCtx; reset(); } bool HuffmanDecoder::reset() { // Default lengths & canonical codes for (uint16 i = 0; i < 256; i++) { _codes[i] = i; _sizes[i] = 8; } memset(_alphabet, 0, sizeof(_alphabet)); memset(_table, 0, sizeof(_table)); return true; } int HuffmanDecoder::readLengths() { const int count = EntropyUtils::decodeAlphabet(_bitstream, _alphabet); if (count == 0) return 0; ExpGolombDecoder egdec(_bitstream, true); int8 curSize = 2; // Read lengths from bitstream for (int i = 0; i < count; i++) { const uint s = _alphabet[i]; if (s > 255) { stringstream ss; ss << "Invalid bitstream: incorrect Huffman symbol " << s; throw BitStreamException(ss.str(), BitStreamException::INVALID_STREAM); } _codes[s] = 0; curSize += int8(egdec.decodeByte()); if ((curSize <= 0) || (curSize > HuffmanCommon::MAX_SYMBOL_SIZE)) { stringstream ss; ss << "Invalid bitstream: incorrect size " << int(curSize); ss << " for Huffman symbol " << s; throw BitStreamException(ss.str(), BitStreamException::INVALID_STREAM); } _sizes[s] = uint16(curSize); } // Create canonical codes if (HuffmanCommon::generateCanonicalCodes(_sizes, _codes, _alphabet, count) < 0) { stringstream ss; ss << "Could not generate Huffman codes: max code length ("; ss << HuffmanCommon::MAX_SYMBOL_SIZE; ss << " bits) exceeded"; throw BitStreamException(ss.str(), BitStreamException::INVALID_STREAM); } return count; } // max(CodeLen) must be <= MAX_SYMBOL_SIZE bool HuffmanDecoder::buildDecodingTable(int count) { // Initialize table with non zero values. // If the bitstream is altered, the decoder may access these default table values. // The number of consumed bits cannot be 0. memset(_table, 7, sizeof(_table)); uint16 length = 0; for (int i = 0; i < count; i++) { const uint s = _alphabet[i]; // All DECODING_BATCH_SIZE bit values read from the bit stream and // starting with the same prefix point to symbol s length = max(_sizes[s], length); const int w = 1 << (DECODING_BATCH_SIZE - length); int idx = int(_codes[s]) * w; const int end = idx + w; if (end > TABLE_MASK + 1) return false; // code -> size, symbol const uint16 val = (uint16(s) << 8) | _sizes[s]; while (idx < end) _table[idx++] = val; } return true; } int HuffmanDecoder::decode(kanzi::byte block[], uint blkptr, uint count) { if (count == 0) return 0; int bsVersion = _pCtx == nullptr ? 6 : _pCtx->getInt("bsVersion", 6); if (bsVersion < 6) return decodeV5(block, blkptr, count); return decodeV6(block, blkptr, count); } int HuffmanDecoder::decodeV6(kanzi::byte block[], uint blkptr, uint count) { const uint minBufSize = 2 * uint(_chunkSize) + (4 * HUFFMAN_FRAGMENT_GUARD_BYTES); if (_bufferSize < minBufSize) { kanzi::byte* buffer = new kanzi::byte[minBufSize]; delete[] _buffer; _buffer = buffer; _bufferSize = minBufSize; } uint startChunk = blkptr; const uint end = blkptr + count; while (startChunk < end) { const uint sizeChunk = min(uint(_chunkSize), end - startChunk); if (sizeChunk < 32) { // Special case for small chunks _bitstream.readBits(&block[startChunk], 8 * sizeChunk); } else { // For each chunk, read code lengths, rebuild codes, rebuild decoding table const int alphabetSize = readLengths(); if (alphabetSize <= 0) return startChunk - blkptr; if (alphabetSize == 1) { // Shortcut for chunks with only one symbol memset(&block[startChunk], _alphabet[0], size_t(sizeChunk)); } else { if (buildDecodingTable(alphabetSize) == false) return -1; if (decodeChunk(&block[startChunk], sizeChunk) == false) return -1; } } startChunk += sizeChunk; } return count; } // count is at least 32 bool HuffmanDecoder::decodeChunk(kanzi::byte block[], uint count) { // Read fragment sizes const int szBits0 = EntropyUtils::readVarInt(_bitstream); const int szBits1 = EntropyUtils::readVarInt(_bitstream); const int szBits2 = EntropyUtils::readVarInt(_bitstream); const int szBits3 = EntropyUtils::readVarInt(_bitstream); if ((szBits0 < 0) || (szBits1 < 0) || (szBits2 < 0) || (szBits3 < 0)) return false; const uint fragCapacity = (_bufferSize - (4 * HUFFMAN_FRAGMENT_GUARD_BYTES)) >> 2; const uint fragStride = fragCapacity + HUFFMAN_FRAGMENT_GUARD_BYTES; const int maxFragBits = int(fragCapacity << 3); if ((szBits0 > maxFragBits) || (szBits1 > maxFragBits) || (szBits2 > maxFragBits) || (szBits3 > maxFragBits)) return false; const int base0 = 0 * fragStride; const int base1 = 1 * fragStride; const int base2 = 2 * fragStride; const int base3 = 3 * fragStride; int idx0 = base0; int idx1 = base1; int idx2 = base2; int idx3 = base3; // Read all compressed data from bitstream _bitstream.readBits(&_buffer[idx0], szBits0); _bitstream.readBits(&_buffer[idx1], szBits1); _bitstream.readBits(&_buffer[idx2], szBits2); _bitstream.readBits(&_buffer[idx3], szBits3); memset(&_buffer[idx0 + ((szBits0 + 7) >> 3)], 0, HUFFMAN_FRAGMENT_GUARD_BYTES); memset(&_buffer[idx1 + ((szBits1 + 7) >> 3)], 0, HUFFMAN_FRAGMENT_GUARD_BYTES); memset(&_buffer[idx2 + ((szBits2 + 7) >> 3)], 0, HUFFMAN_FRAGMENT_GUARD_BYTES); memset(&_buffer[idx3 + ((szBits3 + 7) >> 3)], 0, HUFFMAN_FRAGMENT_GUARD_BYTES); // State variables for each of the four parallel streams uint64 state0 = 0, state1 = 0, state2 = 0, state3 = 0; // bits read from bitstream uint8 bits0 = 0, bits1 = 0, bits2 = 0, bits3 = 0; // number of available bits in state #define READ_STATE(state, idx, bits) do {\ const uint8 shift = (56 - bits) & -8; \ bits += (shift - DECODING_BATCH_SIZE); \ state = (state << shift) | (uint64(BigEndian::readLong64(&_buffer[idx])) >> 1 >> (63 - shift)); /* handle shift = 0 */ \ idx += (shift >> 3); \ } while (0); const int szFrag = count / 4; kanzi::byte* block0 = &block[0 * szFrag]; kanzi::byte* block1 = &block[1 * szFrag]; kanzi::byte* block2 = &block[2 * szFrag]; kanzi::byte* block3 = &block[3 * szFrag]; int n = 0; while (n < szFrag - 4) { // Fill 64 bits of state from the bitstream for each stream READ_STATE(state0, idx0, bits0); READ_STATE(state1, idx1, bits1); READ_STATE(state2, idx2, bits2); READ_STATE(state3, idx3, bits3); // Decompress 4 symbols per stream while keeping the decoded values short-lived. const uint16 val00 = _table[(state0 >> bits0) & TABLE_MASK]; bits0 -= uint8(val00); const uint16 val10 = _table[(state1 >> bits1) & TABLE_MASK]; bits1 -= uint8(val10); const uint16 val20 = _table[(state2 >> bits2) & TABLE_MASK]; bits2 -= uint8(val20); const uint16 val30 = _table[(state3 >> bits3) & TABLE_MASK]; bits3 -= uint8(val30); block0[n + 0] = kanzi::byte(val00 >> 8); block1[n + 0] = kanzi::byte(val10 >> 8); block2[n + 0] = kanzi::byte(val20 >> 8); block3[n + 0] = kanzi::byte(val30 >> 8); const uint16 val01 = _table[(state0 >> bits0) & TABLE_MASK]; bits0 -= uint8(val01); const uint16 val11 = _table[(state1 >> bits1) & TABLE_MASK]; bits1 -= uint8(val11); const uint16 val21 = _table[(state2 >> bits2) & TABLE_MASK]; bits2 -= uint8(val21); const uint16 val31 = _table[(state3 >> bits3) & TABLE_MASK]; bits3 -= uint8(val31); block0[n + 1] = kanzi::byte(val01 >> 8); block1[n + 1] = kanzi::byte(val11 >> 8); block2[n + 1] = kanzi::byte(val21 >> 8); block3[n + 1] = kanzi::byte(val31 >> 8); const uint16 val02 = _table[(state0 >> bits0) & TABLE_MASK]; bits0 -= uint8(val02); const uint16 val12 = _table[(state1 >> bits1) & TABLE_MASK]; bits1 -= uint8(val12); const uint16 val22 = _table[(state2 >> bits2) & TABLE_MASK]; bits2 -= uint8(val22); const uint16 val32 = _table[(state3 >> bits3) & TABLE_MASK]; bits3 -= uint8(val32); block0[n + 2] = kanzi::byte(val02 >> 8); block1[n + 2] = kanzi::byte(val12 >> 8); block2[n + 2] = kanzi::byte(val22 >> 8); block3[n + 2] = kanzi::byte(val32 >> 8); const uint16 val03 = _table[(state0 >> bits0) & TABLE_MASK]; bits0 -= uint8(val03); const uint16 val13 = _table[(state1 >> bits1) & TABLE_MASK]; bits1 -= uint8(val13); const uint16 val23 = _table[(state2 >> bits2) & TABLE_MASK]; bits2 -= uint8(val23); const uint16 val33 = _table[(state3 >> bits3) & TABLE_MASK]; bits3 -= uint8(val33); block0[n + 3] = kanzi::byte(val03 >> 8); block1[n + 3] = kanzi::byte(val13 >> 8); block2[n + 3] = kanzi::byte(val23 >> 8); block3[n + 3] = kanzi::byte(val33 >> 8); bits0 += DECODING_BATCH_SIZE; bits1 += DECODING_BATCH_SIZE; bits2 += DECODING_BATCH_SIZE; bits3 += DECODING_BATCH_SIZE; n += 4; } // Fill 64 bits of state from the bitstream for each stream READ_STATE(state0, idx0, bits0); READ_STATE(state1, idx1, bits1); READ_STATE(state2, idx2, bits2); READ_STATE(state3, idx3, bits3); while (n < szFrag) { // Decompress 1 symbol per stream const uint16 val0 = _table[(state0 >> bits0) & TABLE_MASK]; bits0 -= uint8(val0); const uint16 val1 = _table[(state1 >> bits1) & TABLE_MASK]; bits1 -= uint8(val1); const uint16 val2 = _table[(state2 >> bits2) & TABLE_MASK]; bits2 -= uint8(val2); const uint16 val3 = _table[(state3 >> bits3) & TABLE_MASK]; bits3 -= uint8(val3); block0[n] = kanzi::byte(val0 >> 8); block1[n] = kanzi::byte(val1 >> 8); block2[n] = kanzi::byte(val2 >> 8); block3[n] = kanzi::byte(val3 >> 8); n++; } // Process any remaining bytes at the end of the whole chunk const uint count4 = 4 * szFrag; for (uint i = count4; i < count; i++) block[i] = kanzi::byte(_bitstream.readBits(8)); const int used0 = ((idx0 - base0) << 3) - (int(int8(bits0)) + DECODING_BATCH_SIZE); const int used1 = ((idx1 - base1) << 3) - (int(int8(bits1)) + DECODING_BATCH_SIZE); const int used2 = ((idx2 - base2) << 3) - (int(int8(bits2)) + DECODING_BATCH_SIZE); const int used3 = ((idx3 - base3) << 3) - (int(int8(bits3)) + DECODING_BATCH_SIZE); return (used0 == szBits0) && (used1 == szBits1) && (used2 == szBits2) && (used3 == szBits3); } int HuffmanDecoder::decodeV5(kanzi::byte block[], uint blkptr, uint count) { uint startChunk = blkptr; const uint end = blkptr + count; while (startChunk < end) { const uint endChunk = min(startChunk + _chunkSize, end); const uint sizeChunk = endChunk - startChunk; // For each chunk, read code lengths, rebuild codes, rebuild decoding table const int alphabetSize = readLengths(); if (alphabetSize <= 0) return startChunk - blkptr; if (alphabetSize == 1) { // Shortcut for chunks with only one symbol memset(&block[startChunk], _alphabet[0], size_t(endChunk - startChunk)); startChunk = endChunk; continue; } if (buildDecodingTable(alphabetSize) == false) return -1; // Read number of streams. Only 1 steam supported for now if (_bitstream.readBits(2) != 0) return -1; // Read chunk size const int szBits = EntropyUtils::readVarInt(_bitstream); // A non-empty chunk with multiple symbols cannot have an empty // encoded payload. Treat it as malformed instead of reporting a // successful decode while leaving the destination untouched. if ((szBits <= 0) || (szBits > int(sizeChunk) * HuffmanCommon::MAX_SYMBOL_SIZE)) return -1; // Read compressed data from bitstream { const int sz = (szBits + 7) >> 3; const uint minLenBuf = uint(max(sz + (sz >> 3), 1024)); if (_bufferSize < minLenBuf) { kanzi::byte* buffer = new kanzi::byte[minLenBuf]; delete[] _buffer; _buffer = buffer; _bufferSize = minLenBuf; } _bitstream.readBits(&_buffer[0], szBits); uint64 state = 0; // holds bits read from bitstream uint8 bits = 0; // number of available bits in state int idx = 0; uint n = startChunk; // Decode groups of four only while they fit in the requested // output chunk. A malformed payload can otherwise make the last // group write up to three bytes past endChunk before the final // payload-size check is reached. while ((idx < sz - 8) && (n <= endChunk) && (endChunk - n >= 4)) { const uint8 shift = (56 - bits) & -8; state = (state << shift) | (uint64(BigEndian::readLong64(&_buffer[idx])) >> 1 >> (63 - shift)); // handle shift = 0 idx += (shift >> 3); uint8 bs = bits + shift - DECODING_BATCH_SIZE; const uint16 val0 = _table[(state >> bs) & TABLE_MASK]; bs -= uint8(val0); const uint16 val1 = _table[(state >> bs) & TABLE_MASK]; bs -= uint8(val1); const uint16 val2 = _table[(state >> bs) & TABLE_MASK]; bs -= uint8(val2); const uint16 val3 = _table[(state >> bs) & TABLE_MASK]; bs -= uint8(val3); bits = bs + DECODING_BATCH_SIZE; block[n + 0] = kanzi::byte(val0 >> 8); block[n + 1] = kanzi::byte(val1 >> 8); block[n + 2] = kanzi::byte(val2 >> 8); block[n + 3] = kanzi::byte(val3 >> 8); n += 4; } // Last bytes while (n < endChunk) { while ((bits < HuffmanCommon::MAX_SYMBOL_SIZE) && (idx < sz)) { state = (state << 8) | uint64(_buffer[idx] & kanzi::byte(0xFF)); idx++; // 'bits' may overshoot when idx == sz due to padding state bits // It is necessary to compute proper _table indexes // and has no consequence (except bits != 0 at end of chunk) bits += 8; } // Sanity check if (bits > 64) return n; uint16 val; if (bits >= DECODING_BATCH_SIZE) val = _table[(state >> (bits - DECODING_BATCH_SIZE)) & TABLE_MASK]; else val = _table[(state << (DECODING_BATCH_SIZE - bits)) & TABLE_MASK]; bits -= uint8(val); block[n++] = kanzi::byte(val >> 8); } if (((idx << 3) - int(int8(bits))) != szBits) return -1; } startChunk = endChunk; } return count; } kanzi-cpp-2.6.0/src/entropy/HuffmanDecoder.hpp000066400000000000000000000036121525576310500213000ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_HuffmanDecoder #define knz_HuffmanDecoder #include "HuffmanCommon.hpp" #include "../Context.hpp" #include "../EntropyDecoder.hpp" namespace kanzi { // Implementation of a static Huffman coder. class HuffmanDecoder : public EntropyDecoder { public: HuffmanDecoder(InputBitStream& bitstream, Context* pCtx = nullptr, int chunkSize = HuffmanCommon::MAX_CHUNK_SIZE) ; ~HuffmanDecoder() { _dispose(); if (_buffer != nullptr) delete[] _buffer; } int decode(byte block[], uint blkptr, uint len); InputBitStream& getBitStream() const { return _bitstream; } void dispose() { _dispose(); } private: static const int DECODING_BATCH_SIZE; static const int TABLE_MASK; InputBitStream& _bitstream; byte* _buffer; uint _bufferSize; uint16 _codes[256]; uint _alphabet[256]; uint16 _sizes[256]; uint16 _table[1 << 12]; // decoding table: code -> size, symbol int _chunkSize; Context* _pCtx; int readLengths(); bool decodeChunk(byte block[], uint count); bool buildDecodingTable(int count); bool reset(); int decodeV5(byte block[], uint blkptr, uint len); int decodeV6(byte block[], uint blkptr, uint len); void _dispose() const {} }; } #endif kanzi-cpp-2.6.0/src/entropy/HuffmanEncoder.cpp000066400000000000000000000274061525576310500213140ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include "HuffmanEncoder.hpp" #include "EntropyUtils.hpp" #include "ExpGolombEncoder.hpp" #include "../Global.hpp" #include "../Memory.hpp" using namespace kanzi; using namespace std; // The chunk size indicates how many bytes are encoded (per block) before // resetting the frequency stats. 0 means that frequencies calculated at the // beginning of the block apply to the whole block. HuffmanEncoder::HuffmanEncoder(OutputBitStream& bitstream, int chunkSize) : _bitstream(bitstream) { if (chunkSize < 1024) throw invalid_argument("Huffman codec: The chunk size must be at least 1024"); if (chunkSize > HuffmanCommon::MAX_CHUNK_SIZE) { stringstream ss; ss << "Huffman codec: The chunk size must be at most " << HuffmanCommon::MAX_CHUNK_SIZE; throw invalid_argument(ss.str()); } _chunkSize = chunkSize; _buffer = nullptr; _bufferSize = 0; reset(); } bool HuffmanEncoder::reset() { for (uint16 i = 0; i < 256; i++) _codes[i] = i; return true; } // Rebuild Huffman codes int HuffmanEncoder::updateFrequencies(uint freqs[]) { int count = 0; uint16 sizes[256] = { 0 }; uint alphabet[256] = { 0 }; for (int i = 0; i < 256; i++) { _codes[i] = 0; if (freqs[i] > 0) alphabet[count++] = i; } EntropyUtils::encodeAlphabet(_bitstream, alphabet, 256, count); if (count == 0) return 0; if (count == 1) { _codes[alphabet[0]] = 1 << 12; sizes[alphabet[0]] = 1; } else { uint ranks[256]; // sorted ranks for (int i = 0; i < count; i++) ranks[i] = (freqs[alphabet[i]] << 8) | alphabet[i]; int maxCodeLen = computeCodeLengths(sizes, ranks, count); if (maxCodeLen == 0) throw invalid_argument("Could not generate Huffman codes: invalid code length 0"); if (maxCodeLen > HuffmanCommon::MAX_SYMBOL_SIZE) { maxCodeLen = limitCodeLengths(alphabet, freqs, sizes, ranks, count); if (maxCodeLen == 0) throw invalid_argument("Could not generate Huffman codes: invalid code length 0"); } if (maxCodeLen > HuffmanCommon::MAX_SYMBOL_SIZE) { uint16 n = 0; for (int i = 0; i < count; i++) { _codes[alphabet[i]] = n; sizes[alphabet[i]] = 8; n++; } } else { HuffmanCommon::generateCanonicalCodes(sizes, _codes, ranks, count); } } // Transmit code lengths only, freqs and codes do not matter ExpGolombEncoder egenc(_bitstream, true); uint16 prevSize = 2; // Pack size and code (size <= MAX_SYMBOL_SIZE bits) // Unary encode the code length differences for (int i = 0; i < count; i++) { const int s = alphabet[i]; _codes[s] |= uint16(sizes[s] << 12); egenc.encodeByte(kanzi::byte(sizes[s] - prevSize)); prevSize = sizes[s]; } return count; } int HuffmanEncoder::limitCodeLengths(const uint alphabet[], uint freqs[], uint16 sizes[], uint ranks[], int count) const { int n = 0; int debt = 0; // Fold over-the-limit sizes, skip at-the-limit sizes => incur bit debt while (sizes[ranks[n]] >= HuffmanCommon::MAX_SYMBOL_SIZE) { debt += (sizes[ranks[n]] - HuffmanCommon::MAX_SYMBOL_SIZE); sizes[ranks[n]] = HuffmanCommon::MAX_SYMBOL_SIZE; n++; } if (debt == 0) return HuffmanCommon::MAX_SYMBOL_SIZE; // Check (up to) 6 levels; one fixed-size bucket per size delta. // The alphabet is capped at 256 symbols, so no allocation is needed here. uint8 v[6][256] = { { 0 } }; uint16 vSize[6] = { 0 }; uint16 vHead[6] = { 0 }; while (n < count) { const int idx = HuffmanCommon::MAX_SYMBOL_SIZE - 1 - sizes[ranks[n]]; if ((idx > 5) || (debt < (1 << idx))) break; v[idx][vSize[idx]++] = uint8(n); n++; } int idx = 5; // Repay bit debt in a "semi optimized" way while ((debt > 0) && (idx >= 0)) { if ((vHead[idx] >= vSize[idx]) || (debt < (1 << idx))) { idx--; continue; } // Access element at current head sizes[ranks[v[idx][vHead[idx]]]]++; debt -= (1 << idx); // Advance head vHead[idx]++; } idx = 0; // Adjust if necessary while ((debt > 0) && (idx < 6)) { if (vHead[idx] >= vSize[idx]) { idx++; continue; } sizes[ranks[v[idx][vHead[idx]]]]++; debt -= (1 << idx); vHead[idx]++; } if (debt > 0) { // Fallback to slow (more accurate) path if fast path failed to repay the debt uint alpha[256] = { 0 }; uint f[256]; uint totalFreq = 0; for (int i = 0; i < count; i++) { f[i] = freqs[alphabet[i]]; totalFreq += f[i]; } // Renormalize to a smaller scale EntropyUtils::normalizeFrequencies(f, alpha, count, totalFreq, HuffmanCommon::MAX_CHUNK_SIZE >> 3); for (int i = 0; i < count; i++) { freqs[alphabet[i]] = f[i]; ranks[i] = (f[i] << 8) | alphabet[i]; } return computeCodeLengths(sizes, ranks, count); } return HuffmanCommon::MAX_SYMBOL_SIZE; } // Called only when more than 1 symbol int HuffmanEncoder::computeCodeLengths(uint16 sizes[], uint ranks[], int count) const { // Sort ranks by increasing freqs (first key) and increasing value (second key) sort(ranks, ranks + count); uint freqs[256] = { 0 }; bool valid = true; for (int i = 0; i < count; i++) { freqs[i] = ranks[i] >> 8; ranks[i] = ranks[i] & 0xFF; valid &= (freqs[i] != 0); } if (valid == false) return 0; // See [In-Place Calculation of Minimum-Redundancy Codes] // by Alistair Moffat & Jyrki Katajainen computeInPlaceSizesPhase1(freqs, count); const int maxCodeLen = computeInPlaceSizesPhase2(freqs, count); for (int i = 0; i < count; i++) sizes[ranks[i]] = uint16(freqs[i]); return maxCodeLen; } void HuffmanEncoder::computeInPlaceSizesPhase1(uint data[], int n) { for (int s = 0, r = 0, t = 0; t < n - 1; t++) { uint sum = 0; for (int i = 0; i < 2; i++) { if ((s >= n) || ((r < t) && (data[r] < data[s]))) { sum += data[r]; data[r] = t; r++; continue; } sum += data[s]; if (s > t) data[s] = 0; s++; } data[t] = sum; } } // n must be at least 2 // return max symbol length uint HuffmanEncoder::computeInPlaceSizesPhase2(uint data[], int n) { if (n < 2) return 0; uint topLevel = n - 2; //root uint depth = 1; uint totalNodesAtLevel = 2; while (n > 0) { uint k = topLevel; while ((k != 0) && (data[k - 1] >= topLevel)) k--; const int internalNodesAtLevel = topLevel - k; const int leavesAtLevel = totalNodesAtLevel - internalNodesAtLevel; for (int j = 0; j < leavesAtLevel; j++) data[--n] = depth; totalNodesAtLevel = internalNodesAtLevel << 1; topLevel = k; depth++; } return depth - 1; } // Dynamically compute the frequencies for every chunk of data in the block int HuffmanEncoder::encode(const kanzi::byte block[], uint blkptr, uint count) { if (count == 0) return 0; const uint sz = uint(_chunkSize); const uint minLenBuf = max(min(sz + (sz >> 3), 2 * count), uint(65536)); if (_bufferSize < minLenBuf) { kanzi::byte* buffer = new kanzi::byte[minLenBuf]; delete[] _buffer; _buffer = buffer; _bufferSize = minLenBuf; } uint startChunk = blkptr; const uint end = startChunk + count; while (startChunk < end) { // Update frequencies and rebuild Huffman codes const uint sizeChunk = min(uint(_chunkSize), end - startChunk); if (sizeChunk < 32) { // Special case for small chunks _bitstream.writeBits(&block[startChunk], 8 * sizeChunk); } else { uint freqs[256] = { 0 }; Global::computeHistogram(&block[startChunk], sizeChunk, freqs); // Skip chunk if only one symbol if (updateFrequencies(freqs) > 1) { encodeChunk(&block[startChunk], sizeChunk); } } startChunk += sizeChunk; } return count; } // count is at least 32 void HuffmanEncoder::encodeChunk(const kanzi::byte block[], uint count) { uint nbBits[4] = { 0 }; const uint szFrag = count / 4; const uint szFrag4 = szFrag & ~3; const uint szBuf = _bufferSize / 4; // Encode chunk for (int j = 0; j < 4; j++) { const kanzi::byte* src = &block[j * szFrag]; kanzi::byte* buf = &_buffer[j * szBuf]; int idx = 0; int bits = 0; // number of accumulated bits uint64 state = 0; // Encode fragments sequentially for (uint i = 0; i < szFrag4; i += 4) { const uint16 code0 = _codes[int(src[i])]; const uint16 codeLen0 = code0 >> 12; const uint16 code1 = _codes[int(src[i + 1])]; const uint16 codeLen1 = code1 >> 12; const uint16 code2 = _codes[int(src[i + 2])]; const uint16 codeLen2 = code2 >> 12; const uint16 code3 = _codes[int(src[i + 3])]; const uint16 codeLen3 = code3 >> 12; const uint32 pack01 = (uint32(code0 & 0x0FFF) << codeLen1) | uint32(code1 & 0x0FFF); const uint32 pack23 = (uint32(code2 & 0x0FFF) << codeLen3) | uint32(code3 & 0x0FFF); state = (state << (codeLen0 + codeLen1)) | uint64(pack01); state = (state << (codeLen2 + codeLen3)) | uint64(pack23); bits += (codeLen0 + codeLen1 + codeLen2 + codeLen3); BigEndian::writeLong64(&buf[idx], state << (64 - bits)); // bits cannot be 0 idx += (bits >> 3); bits &= 7; } // Fragment last bytes for (uint i = szFrag4; i < szFrag; i++) { const uint16 code = _codes[int(src[i])]; const uint16 codeLen = code >> 12; state = (state << codeLen) | uint64(code & 0x0FFF); bits += codeLen; } nbBits[j] = (idx * 8) + bits; while (bits >= 8) { bits -= 8; buf[idx++] = kanzi::byte(state >> bits); } if (bits > 0) buf[idx++] = kanzi::byte(state << (8 - bits)); } // Write chunk size in bits EntropyUtils::writeVarInt(_bitstream, nbBits[0]); EntropyUtils::writeVarInt(_bitstream, nbBits[1]); EntropyUtils::writeVarInt(_bitstream, nbBits[2]); EntropyUtils::writeVarInt(_bitstream, nbBits[3]); // Write compressed data to bitstream _bitstream.writeBits(&_buffer[0 * szBuf], nbBits[0]); _bitstream.writeBits(&_buffer[1 * szBuf], nbBits[1]); _bitstream.writeBits(&_buffer[2 * szBuf], nbBits[2]); _bitstream.writeBits(&_buffer[3 * szBuf], nbBits[3]); // Chunk last bytes const uint count4 = 4 * szFrag; for (uint i = count4; i < count; i++) _bitstream.writeBits(uint64(block[i]), 8); } kanzi-cpp-2.6.0/src/entropy/HuffmanEncoder.hpp000066400000000000000000000036031525576310500213120ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_HuffmanEncoder #define knz_HuffmanEncoder #include "HuffmanCommon.hpp" #include "../EntropyEncoder.hpp" namespace kanzi { // Implementation of a static Huffman encoder. // Uses in place generation of canonical codes instead of a tree class HuffmanEncoder : public EntropyEncoder { public: HuffmanEncoder(OutputBitStream& bitstream, int chunkSize = HuffmanCommon::MAX_CHUNK_SIZE); ~HuffmanEncoder() { _dispose(); if (_buffer != nullptr) delete[] _buffer; } int updateFrequencies(uint frequencies[]); int encode(const byte block[], uint blkptr, uint len); OutputBitStream& getBitStream() const { return _bitstream; } void dispose() { _dispose(); } private: OutputBitStream& _bitstream; uint16 _codes[256]; int _chunkSize; byte* _buffer; uint _bufferSize; void encodeChunk(const byte block[], uint count); int computeCodeLengths(uint16 sizes[], uint sranks[], int count) const; int limitCodeLengths(const uint alphabet[], uint freqs[], uint16 sizes[], uint sranks[], int count) const; void _dispose() const {} bool reset(); static void computeInPlaceSizesPhase1(uint data[], int n); static uint computeInPlaceSizesPhase2(uint data[], int n); }; } #endif kanzi-cpp-2.6.0/src/entropy/NullEntropyDecoder.hpp000066400000000000000000000035271525576310500222140ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_NullEntropyDecoder #define knz_NullEntropyDecoder #include "../EntropyDecoder.hpp" #include "../InputBitStream.hpp" namespace kanzi { // Null entropy decoder // Pass through that writes the data directly to the bitstream class NullEntropyDecoder FINAL : public EntropyDecoder { private: InputBitStream& _bitstream; public: NullEntropyDecoder(InputBitStream& bitstream); ~NullEntropyDecoder() {} int decode(byte block[], uint blkptr, uint len); byte decodeByte(); InputBitStream& getBitStream() const { return _bitstream; } void dispose() {} }; inline NullEntropyDecoder::NullEntropyDecoder(InputBitStream& bitstream) : _bitstream(bitstream) { } inline int NullEntropyDecoder::decode(byte block[], uint blkptr, uint count) { uint res = 0; while (count != 0) { const uint ckSize = (count < 1<<23) ? count : 1<<23; const uint r = uint(_bitstream.readBits(&block[blkptr], 8 * ckSize) >> 3); if (r == 0) break; res += r; blkptr += r; count -= r; } return res; } inline byte NullEntropyDecoder::decodeByte() { return byte(_bitstream.readBits(8)); } } #endif kanzi-cpp-2.6.0/src/entropy/NullEntropyEncoder.hpp000066400000000000000000000035721525576310500222260ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_NullEntropyEncoder #define knz_NullEntropyEncoder #include "../EntropyEncoder.hpp" #include "../OutputBitStream.hpp" namespace kanzi { // Null entropy encoder // Pass through that writes the data directly to the bitstream class NullEntropyEncoder FINAL : public EntropyEncoder { private: OutputBitStream& _bitstream; public: NullEntropyEncoder(OutputBitStream& bitstream); ~NullEntropyEncoder() {} int encode(const byte block[], uint blkptr, uint len); void encodeByte(byte val); OutputBitStream& getBitStream() const { return _bitstream; } void dispose() {} }; inline NullEntropyEncoder::NullEntropyEncoder(OutputBitStream& bitstream) : _bitstream(bitstream) { } inline int NullEntropyEncoder::encode(const byte block[], uint blkptr, uint count) { uint res = 0; while (count != 0) { const uint ckSize = (count < 1<<23) ? count : 1<<23; const uint w = uint(_bitstream.writeBits(&block[blkptr], 8 * ckSize) >> 3); if (w == 0) break; res += w; blkptr += w; count -= w; } return res; } inline void NullEntropyEncoder::encodeByte(byte val) { _bitstream.writeBits(uint64(val), 8); } } #endif kanzi-cpp-2.6.0/src/entropy/RangeDecoder.cpp000066400000000000000000000152161525576310500207460ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include "../BitStreamException.hpp" #include "RangeDecoder.hpp" #include "EntropyUtils.hpp" using namespace kanzi; using namespace std; const int RangeDecoder::DECODING_BATCH_SIZE = 12; // in bits const int RangeDecoder::DECODING_MASK = (1 << DECODING_BATCH_SIZE) - 1; const uint64 RangeDecoder::TOP_RANGE = 0x0FFFFFFFFFFFFFFF; const uint64 RangeDecoder::BOTTOM_RANGE = 0x000000000000FFFF; const uint64 RangeDecoder::RANGE_MASK = 0x0FFFFFFF00000000; const int RangeDecoder::DEFAULT_CHUNK_SIZE = 1 << 15; // 32 KB by default const int RangeDecoder::DEFAULT_LOG_RANGE = 12; const int RangeDecoder::MAX_CHUNK_SIZE = 1 << 30; // The chunk size indicates how many bytes are encoded (per block) before // resetting the frequency stats. RangeDecoder::RangeDecoder(InputBitStream& bitstream, int chunkSize) : _bitstream(bitstream) { if (chunkSize < 1024) throw invalid_argument("The chunk size must be at least 1024"); if (chunkSize > MAX_CHUNK_SIZE) throw invalid_argument("The chunk size must be at most 2^30"); _f2s = nullptr; _chunkSize = chunkSize; reset(); } bool RangeDecoder::reset() { _low = 0; _range = TOP_RANGE; _code = 0; _lenF2S = 0; _shift = 0; memset(_alphabet, 0, sizeof(uint) * 256); memset(_freqs, 0, sizeof(uint) * 256); memset(_cumFreqs, 0, sizeof(uint64) * 257); return true; } int RangeDecoder::decodeHeader(uint frequencies[]) { int alphabetSize = EntropyUtils::decodeAlphabet(_bitstream, _alphabet); if (alphabetSize == 0) return 0; if (alphabetSize != 256) { memset(frequencies, 0, sizeof(uint) * 256); } const uint logRange = uint(8 + _bitstream.readBits(3)); const int scale = 1 << logRange; _shift = logRange; int sum = 0; const int chkSize = (alphabetSize >= 64) ? 8 : 6; int llr = 3; while (uint(1 << llr) <= logRange) llr++; // Decode all frequencies (but the first one) by chunks of size 'inc' for (int i = 1; i < alphabetSize; i += chkSize) { const int logMax = int(_bitstream.readBits(llr)); if ((1 << logMax) > scale) { stringstream ss; ss << "Invalid bitstream: incorrect frequency size "; ss << logMax << " in ANS range decoder"; throw BitStreamException(ss.str(), BitStreamException::INVALID_STREAM); } const int endj = min(i + chkSize, alphabetSize); // Read frequencies for (int j = i; j < endj; j++) { const int freq = (logMax == 0) ? 1 : int(_bitstream.readBits(logMax) + 1); if ((freq <= 0) || (freq >= scale)) { stringstream ss; ss << "Invalid bitstream: incorrect frequency " << freq; ss << " for symbol '" << _alphabet[j] << "' in range decoder"; throw BitStreamException(ss.str(), BitStreamException::INVALID_STREAM); } frequencies[_alphabet[j]] = uint(freq); sum += freq; } } // Infer first frequency if (scale <= sum) { stringstream ss; ss << "Invalid bitstream: incorrect frequency " << frequencies[_alphabet[0]]; ss << " for symbol '" << _alphabet[0] << "' in range decoder"; throw BitStreamException(ss.str(), BitStreamException::INVALID_STREAM); } frequencies[_alphabet[0]] = uint(scale - sum); _cumFreqs[0] = 0; if (_lenF2S < scale) { short* f2s = new short[scale]; delete[] _f2s; _f2s = f2s; _lenF2S = scale; } // Create histogram of frequencies scaled to 'range' and reverse mapping for (int i = 0; i < 256; i++) { _cumFreqs[i + 1] = _cumFreqs[i] + frequencies[i]; const int base = int(_cumFreqs[i]); for (int j = frequencies[i] - 1; j >= 0; j--) _f2s[base + j] = short(i); } return alphabetSize; } // Initialize once (if necessary) at the beginning, the use the faster decodeByte_() // Reset frequency stats for each chunk of data in the block int RangeDecoder::decode(kanzi::byte block[], uint blkptr, uint count) { if (count == 0) return 0; const uint end = blkptr + count; const uint sz = _chunkSize; uint startChunk = blkptr; while (startChunk < end) { const uint endChunk = min(startChunk + sz, end); const int alphabetSize = decodeHeader(_freqs); if (alphabetSize == 0) return startChunk - blkptr; if (alphabetSize == 1) { // Shortcut for chunks with only one symbol memset(&block[startChunk], _alphabet[0], size_t(endChunk - startChunk)); startChunk = endChunk; continue; } _range = TOP_RANGE; _low = 0; _code = _bitstream.readBits(60); for (uint i = startChunk; i < endChunk; i++) block[i] = decodeByte(); startChunk = endChunk; } return count; } kanzi::byte RangeDecoder::decodeByte() { // Compute next low and range _range >>= _shift; if (_range == 0) { throw BitStreamException("Invalid bitstream: incorrect range in range decoder", BitStreamException::INVALID_STREAM); } const uint64 cum = (_code - _low) / _range; if (cum >= (uint64(1) << _shift)) { throw BitStreamException("Invalid bitstream: incorrect cumulative frequency in range decoder", BitStreamException::INVALID_STREAM); } const int symbol = _f2s[int(cum)]; const uint64 cumFreq = _cumFreqs[symbol]; const uint64 freq = _cumFreqs[symbol + 1] - cumFreq; _low += (cumFreq * _range); _range *= freq; // If the left-most digits are the same throughout the range, read bits from bitstream while (true) { if (((_low ^ (_low + _range)) & RANGE_MASK) != 0) { if (_range > BOTTOM_RANGE) break; // Normalize _range = ~(_low-1) & BOTTOM_RANGE; } _code = (_code << 28) | _bitstream.readBits(28); _range <<= 28; _low <<= 28; } return kanzi::byte(symbol); } kanzi-cpp-2.6.0/src/entropy/RangeDecoder.hpp000066400000000000000000000040521525576310500207470ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_RangeDecoder #define knz_RangeDecoder #include "../EntropyDecoder.hpp" namespace kanzi { // Based on Order 0 range coder by Dmitry Subbotin itself derived from the algorithm // described by G.N.N Martin in his seminal article in 1979. // [G.N.N. Martin on the Data Recording Conference, Southampton, 1979] // Optimized for speed. class RangeDecoder : public EntropyDecoder { public: static const int DECODING_BATCH_SIZE; static const int DECODING_MASK; RangeDecoder(InputBitStream& bitstream, int chunkSize = DEFAULT_CHUNK_SIZE); ~RangeDecoder() { _dispose(); if (_f2s != nullptr) delete[] _f2s; } int decode(byte block[], uint blkptr, uint len); InputBitStream& getBitStream() const { return _bitstream; } void dispose() { _dispose(); } private: static const uint64 TOP_RANGE; static const uint64 BOTTOM_RANGE; static const uint64 RANGE_MASK; static const int DEFAULT_CHUNK_SIZE; static const int DEFAULT_LOG_RANGE; static const int MAX_CHUNK_SIZE; uint64 _code; uint64 _low; uint64 _range; uint _alphabet[256]; uint _freqs[256]; uint64 _cumFreqs[257]; short* _f2s; int _lenF2S; InputBitStream& _bitstream; uint _chunkSize; uint _shift; int decodeHeader(uint frequencies[]); byte decodeByte(); bool reset(); void _dispose() const {} }; } #endif kanzi-cpp-2.6.0/src/entropy/RangeEncoder.cpp000066400000000000000000000134301525576310500207540ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include "RangeEncoder.hpp" #include "EntropyUtils.hpp" #include "../Global.hpp" using namespace kanzi; using namespace std; const uint64 RangeEncoder::TOP_RANGE = 0x0FFFFFFFFFFFFFFF; const uint64 RangeEncoder::BOTTOM_RANGE = 0x000000000000FFFF; const uint64 RangeEncoder::RANGE_MASK = 0x0FFFFFFF00000000; const int RangeEncoder::DEFAULT_CHUNK_SIZE = 1 << 15; // 32 KB by default const int RangeEncoder::DEFAULT_LOG_RANGE = 12; const int RangeEncoder::MAX_CHUNK_SIZE = 1 << 30; // The chunk size indicates how many bytes are encoded (per block) before // resetting the frequency stats. RangeEncoder::RangeEncoder(OutputBitStream& bitstream, int chunkSize, int logRange) : _bitstream(bitstream) { if (chunkSize < 1024) throw invalid_argument("The chunk size must be at least 1024"); if (chunkSize > MAX_CHUNK_SIZE) throw invalid_argument("The chunk size must be at most 2^30"); if ((logRange < 8) || (logRange > 15)) { stringstream ss; ss << "Invalid range parameter: " << logRange << " (must be in [8..15])"; throw invalid_argument(ss.str()); } _logRange = logRange; _chunkSize = chunkSize; reset(); } bool RangeEncoder::reset() { _low = 0; _range = TOP_RANGE; _shift = 0; memset(_alphabet, 0, 256 * sizeof(uint)); memset(_freqs, 0, 256 * sizeof(uint)); memset(_cumFreqs, 0, 257 * sizeof(uint64)); return true; } int RangeEncoder::updateFrequencies(uint frequencies[], int size, int lr) { int alphabetSize = EntropyUtils::normalizeFrequencies(frequencies, _alphabet, 256, size, 1 << lr); if (alphabetSize > 0) { _cumFreqs[0] = 0; // Create histogram of frequencies scaled to 'range' for (int i = 0; i < 256; i++) _cumFreqs[i + 1] = _cumFreqs[i] + frequencies[i]; } encodeHeader(alphabetSize, _alphabet, frequencies, lr); return alphabetSize; } bool RangeEncoder::encodeHeader(int alphabetSize, const uint alphabet[], const uint frequencies[], int lr) const { const int encoded = EntropyUtils::encodeAlphabet(_bitstream, alphabet, 256, alphabetSize); if (encoded < 0) return false; if (encoded == 0) return true; _bitstream.writeBits(lr - 8, 3); // logRange if (encoded == 1) return true; int chkSize = (alphabetSize >= 64) ? 8 : 6; int llr = 3; while (1 << llr <= lr) llr++; // Encode all frequencies (but the first one) by chunks for (int i = 1; i < alphabetSize; i += chkSize) { uint max = frequencies[alphabet[i]] - 1; int endj = min(i + chkSize, alphabetSize); // Search for max frequency log size in next chunk for (int j = i + 1; j < endj; j++) { if (frequencies[alphabet[j]] - 1 > max) max = frequencies[alphabet[j]] - 1; } const uint logMax = (max == 0) ? 0 : Global::_log2(max) + 1; _bitstream.writeBits(logMax, llr); if (logMax == 0) // all frequencies equal one in this chunk continue; // Write frequencies for (int j = i; j < endj; j++) _bitstream.writeBits(frequencies[alphabet[j]] - 1, logMax); } return true; } // Reset frequency stats for each chunk of data in the block int RangeEncoder::encode(const kanzi::byte block[], uint blkptr, uint count) { if (count == 0) return 0; const uint end = blkptr + count; const uint sz = _chunkSize; uint startChunk = blkptr; while (startChunk < end) { const uint endChunk = min(startChunk + sz, end); _range = TOP_RANGE; _low = 0; int lr = _logRange; // Lower log range if the size of the data chunk is small while ((lr > 8) && (uint(1 << lr) > endChunk - startChunk)) lr--; if (rebuildStatistics(block, startChunk, endChunk, lr) <= 1) { // Skip chunk if only one symbol startChunk = endChunk; continue; } _shift = lr; for (uint i = startChunk; i < endChunk; i++) encodeByte(block[i]); // Flush 'low' _bitstream.writeBits(_low, 60); startChunk = endChunk; } return count; } void RangeEncoder::encodeByte(kanzi::byte b) { // Compute next low and range const int symbol = int(b); const uint64 cumFreq = _cumFreqs[symbol]; const uint64 freq = _cumFreqs[symbol + 1] - cumFreq; _range >>= _shift; _low += (cumFreq * _range); _range *= freq; // If the left-most digits are the same throughout the range, write bits to bitstream while (true) { if (((_low ^ (_low + _range)) & RANGE_MASK) != 0) { if (_range > BOTTOM_RANGE) break; // Normalize _range = ~(_low - 1) & BOTTOM_RANGE; } _bitstream.writeBits(_low >> 32, 28); _range <<= 28; _low <<= 28; } } // Compute chunk frequencies, cumulated frequencies and encode chunk header int RangeEncoder::rebuildStatistics(const kanzi::byte block[], int start, int end, int lr) { memset(_freqs, 0, sizeof(_freqs)); Global::computeHistogram(&block[start], end - start, _freqs); return updateFrequencies(_freqs, end - start, lr); } kanzi-cpp-2.6.0/src/entropy/RangeEncoder.hpp000066400000000000000000000042161525576310500207630ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_RangeEncoder #define knz_RangeEncoder #include "../EntropyEncoder.hpp" namespace kanzi { // Based on Order 0 range coder by Dmitry Subbotin itself derived from the algorithm // described by G.N.N Martin in his seminal article in 1979. // [G.N.N. Martin on the Data Recording Conference, Southampton, 1979] // Optimized for speed. class RangeEncoder : public EntropyEncoder { public: RangeEncoder(OutputBitStream& bitstream, int chunkSize = DEFAULT_CHUNK_SIZE, int logRange=DEFAULT_LOG_RANGE); ~RangeEncoder() { _dispose(); } int encode(const byte block[], uint blkptr, uint len); OutputBitStream& getBitStream() const { return _bitstream; } void dispose() { _dispose(); } private: static const uint64 TOP_RANGE; static const uint64 BOTTOM_RANGE; static const uint64 RANGE_MASK; static const int DEFAULT_CHUNK_SIZE; static const int DEFAULT_LOG_RANGE; static const int MAX_CHUNK_SIZE; uint64 _low; uint64 _range; uint _alphabet[256]; uint _freqs[256]; uint64 _cumFreqs[257]; OutputBitStream& _bitstream; uint _chunkSize; uint _logRange; uint _shift; int rebuildStatistics(const byte block[], int start, int end, int lr); int updateFrequencies(uint frequencies[], int size, int lr); void encodeByte(byte b); bool encodeHeader(int alphabetSize, const uint alphabet[], const uint frequencies[], int lr) const; bool reset(); void _dispose() const {} }; } #endif kanzi-cpp-2.6.0/src/entropy/TPAQPredictor.cpp000066400000000000000000000035671525576310500210530ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include "TPAQPredictor.hpp" using namespace kanzi; const int TPAQMixer::BEGIN_LEARN_RATE = 60 << 7; const int TPAQMixer::END_LEARN_RATE = 11 << 7; template<> const int TPAQPredictor::MAX_LENGTH = 88; template<> const int TPAQPredictor::BUFFER_SIZE = 64 * 1024 * 1024; template<> const int TPAQPredictor::HASH_SIZE = 16 * 1024 * 1024; template<> const int TPAQPredictor::HASH = 0x7FEB352D; template<> const uint TPAQPredictor::MASK_80808080 = 0x80808080u; template<> const uint TPAQPredictor::MASK_F0F0F000 = 0xF0F0F000u; template<> const uint TPAQPredictor::MASK_4F4FFFFF = 0x4F4FFFFFu; template<> const int TPAQPredictor::MAX_LENGTH = 88; template<> const int TPAQPredictor::BUFFER_SIZE = 64 * 1024 * 1024; template<> const int TPAQPredictor::HASH_SIZE = 16 * 1024 * 1024; template<> const int TPAQPredictor::HASH = 0x7FEB352D; template<> const uint TPAQPredictor::MASK_80808080 = 0x80808080u; template<> const uint TPAQPredictor::MASK_F0F0F000 = 0xF0F0F000u; template<> const uint TPAQPredictor::MASK_4F4FFFFF = 0x4F4FFFFFu; TPAQMixer::TPAQMixer() { _pr = 2048; _skew = 0; _w0 = _w1 = _w2 = _w3 = _w4 = _w5 = _w6 = _w7 = 32768; _p0 = _p1 = _p2 = _p3 = _p4 = _p5 = _p6 = _p7 = 0; _learnRate = BEGIN_LEARN_RATE; } kanzi-cpp-2.6.0/src/entropy/TPAQPredictor.hpp000066400000000000000000000542251525576310500210550ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_TPAQPredictor #define knz_TPAQPredictor #include #include "../Context.hpp" #include "../Global.hpp" #include "../Predictor.hpp" #include "../Memory.hpp" #include "AdaptiveProbMap.hpp" namespace kanzi { // TPAQ predictor // Initially based on Tangelo 2.4 (by Jan Ondrus). // PAQ8 is written by Matt Mahoney. // See http://encode.su/threads/1738-TANGELO-new-compressor-(derived-from-PAQ8-FP8) // Mixer combines models using neural networks with 8 inputs. class TPAQMixer { public: TPAQMixer(); ~TPAQMixer() { } void update(int bit); int get(int p0, int p1, int p2, int p3, int p4, int p5, int p6, int p7); private: static const int BEGIN_LEARN_RATE; static const int END_LEARN_RATE; int _w0, _w1, _w2, _w3, _w4, _w5, _w6, _w7; int _p0, _p1, _p2, _p3, _p4, _p5, _p6, _p7; int _pr; int _skew; int _learnRate; }; template class TPAQPredictor FINAL : public Predictor { public: TPAQPredictor(Context* ctx = nullptr); ~TPAQPredictor(); void update(int bit); // Return the split value representing the probability of 1 in the [0..4095] range. int get() { return _pr; } private: static const int MAX_LENGTH; static const int BUFFER_SIZE; static const int HASH_SIZE; static const int HASH; static const uint MASK_80808080; static const uint MASK_F0F0F000; static const uint MASK_4F4FFFFF; #define SSE0_RATE(T) ((T == true) ? 6 : 7) int _pr; // next predicted value (0-4095) uint _c0; // bitwise context: last 0-7 bits with a leading 1 (1-255) uint _c4; // last 4 whole bytes, last is in low 8 bits uint _c8; // last 8 to 4 whole bytes, last is in low 8 bits int _bpos; // number of bits in c0 (0-7) int _pos; int _binCount; int _matchLen; int _matchPos; int _matchVal; uint _hash; LogisticAdaptiveProbMap _sse0; LogisticAdaptiveProbMap _sse1; TPAQMixer* _mixers; TPAQMixer* _mixer; // current mixer byte* _buffer; int* _hashes; // hash table(context, buffer position) uint8* _bigStatesMap;// hash table(context, prediction) uint8* _smallStatesMap0; // hash table(context, prediction) uint8* _smallStatesMap1; // hash table(context, prediction) uint _statesMask; uint _mixersMask; uint _hashMask; uint _bufferMask; uint8* _cp0; // context pointers uint8* _cp1; uint8* _cp2; uint8* _cp3; uint8* _cp4; uint8* _cp5; uint8* _cp6; int _ctx0; // contexts int _ctx1; int _ctx2; int _ctx3; int _ctx4; int _ctx5; int _ctx6; int hash(uint x, uint y) const; int createContext(uint ctxId, uint cx) const; int getMatchContextPred(); void findMatch(); bool reset(); }; // Adjust weights to minimize coding cost of last prediction inline void TPAQMixer::update(int bit) { const int err = (((bit << 12) - _pr) * _learnRate) >> 10; if (err == 0) return; // Quickly decaying learn rate _learnRate -= (uint32(END_LEARN_RATE - _learnRate) >> 31); _skew += err; // Train Neural Network: update weights _w0 += ((_p0 * err + 0) >> 12); _w1 += ((_p1 * err + 0) >> 12); _w2 += ((_p2 * err + 0) >> 12); _w3 += ((_p3 * err + 0) >> 12); _w4 += ((_p4 * err + 0) >> 12); _w5 += ((_p5 * err + 0) >> 12); _w6 += ((_p6 * err + 0) >> 12); _w7 += ((_p7 * err + 0) >> 12); } inline int TPAQMixer::get(int p0, int p1, int p2, int p3, int p4, int p5, int p6, int p7) { _p0 = p0; _p1 = p1; _p2 = p2; _p3 = p3; _p4 = p4; _p5 = p5; _p6 = p6; _p7 = p7; // Neural Network dot product (sum weights*inputs) _pr = Global::squash(((p0 * _w0) + (p1 * _w1) + (p2 * _w2) + (p3 * _w3) + (p4 * _w4) + (p5 * _w5) + (p6 * _w6) + (p7 * _w7) + _skew + 65536) >> 17); return _pr; } ///////////////////////// state table //////////////////////// // States represent a bit history within some context. // State 0 is the starting state (no bits seen). // States 1-30 represent all possible sequences of 1-4 bits. // States 31-252 represent a pair of counts, (n0,n1), the number // of 0 and 1 bits respectively. If n0+n1 < 16 then there are // two states for each pair, depending on if a 0 or 1 was the last // bit seen. // If n0 and n1 are too large, then there is no state to represent this // pair, so another state with about the same ratio of n0/n1 is substituted. // Also, when a bit is observed and the count of the opposite bit is large, // then part of this count is discarded to favor newer data over old. const uint8 STATE_TRANSITIONS[2][256] = { // Bit 0 { 1, 3, 143, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 47, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 6, 71, 71, 71, 61, 75, 56, 77, 78, 77, 80, 81, 82, 83, 84, 85, 86, 87, 88, 77, 90, 91, 92, 80, 94, 95, 96, 97, 98, 99, 90, 101, 94, 103, 101, 102, 104, 107, 104, 105, 108, 111, 112, 113, 114, 115, 116, 92, 118, 94, 103, 119, 122, 123, 94, 113, 126, 113, 128, 129, 114, 131, 132, 112, 134, 111, 134, 110, 134, 134, 128, 128, 142, 143, 115, 113, 142, 128, 148, 149, 79, 148, 142, 148, 150, 155, 149, 157, 149, 159, 149, 131, 101, 98, 115, 114, 91, 79, 58, 1, 170, 129, 128, 110, 174, 128, 176, 129, 174, 179, 174, 176, 141, 157, 179, 185, 157, 187, 188, 168, 151, 191, 192, 188, 187, 172, 175, 170, 152, 185, 170, 176, 170, 203, 148, 185, 203, 185, 192, 209, 188, 211, 192, 213, 214, 188, 216, 168, 84, 54, 54, 221, 54, 55, 85, 69, 63, 56, 86, 58, 230, 231, 57, 229, 56, 224, 54, 54, 66, 58, 54, 61, 57, 222, 78, 85, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, // Bit 1 { 2, 163, 169, 163, 165, 89, 245, 217, 245, 245, 233, 244, 227, 74, 221, 221, 218, 226, 243, 218, 238, 242, 74, 238, 241, 240, 239, 224, 225, 221, 232, 72, 224, 228, 223, 225, 238, 73, 167, 76, 237, 234, 231, 72, 31, 63, 225, 237, 236, 235, 53, 234, 53, 234, 229, 219, 229, 233, 232, 228, 226, 72, 74, 222, 75, 220, 167, 57, 218, 70, 168, 72, 73, 74, 217, 76, 167, 79, 79, 166, 162, 162, 162, 162, 165, 89, 89, 165, 89, 162, 93, 93, 93, 161, 100, 93, 93, 93, 93, 93, 161, 102, 120, 104, 105, 106, 108, 106, 109, 110, 160, 134, 108, 108, 126, 117, 117, 121, 119, 120, 107, 124, 117, 117, 125, 127, 124, 139, 130, 124, 133, 109, 110, 135, 110, 136, 137, 138, 127, 140, 141, 145, 144, 124, 125, 146, 147, 151, 125, 150, 127, 152, 153, 154, 156, 139, 158, 139, 156, 139, 130, 117, 163, 164, 141, 163, 147, 2, 2, 199, 171, 172, 173, 177, 175, 171, 171, 178, 180, 172, 181, 182, 183, 184, 186, 178, 189, 181, 181, 190, 193, 182, 182, 194, 195, 196, 197, 198, 169, 200, 201, 202, 204, 180, 205, 206, 207, 208, 210, 194, 212, 184, 215, 193, 184, 208, 193, 163, 219, 168, 94, 217, 223, 224, 225, 76, 227, 217, 229, 219, 79, 86, 165, 217, 214, 225, 216, 216, 234, 75, 214, 237, 74, 74, 163, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } }; const int STATE_MAP[] = { -31, -400, 406, -547, -642, -743, -827, -901, -901, -974, -945, -955, -1060, -1031, -1044, -956, -994, -1035, -1147, -1069, -1111, -1145, -1096, -1084, -1171, -1199, -1062, -1498, -1199, -1199, -1328, -1405, -1275, -1248, -1167, -1448, -1441, -1199, -1357, -1160, -1437, -1428, -1238, -1343, -1526, -1331, -1443, -2047, -2047, -2044, -2047, -2047, -2047, -232, -414, -573, -517, -768, -627, -666, -644, -740, -721, -829, -770, -963, -863, -1099, -811, -830, -277, -1036, -286, -218, -42, -411, 141, -1014, -1028, -226, -469, -540, -573, -581, -594, -610, -628, -711, -670, -144, -408, -485, -464, -173, -221, -310, -335, -375, -324, -413, -99, -179, -105, -150, -63, -9, 56, 83, 119, 144, 198, 118, -42, -96, -188, -285, -376, 107, -138, 38, -82, 186, -114, -190, 200, 327, 65, 406, 108, -95, 308, 171, -18, 343, 135, 398, 415, 464, 514, 494, 508, 519, 92, -123, 343, 575, 585, 516, -7, -156, 209, 574, 613, 621, 670, 107, 989, 210, 961, 246, 254, -12, -108, 97, 281, -143, 41, 173, -209, 583, -55, 250, 354, 558, 43, 274, 14, 488, 545, 84, 528, 519, 587, 634, 663, 95, 700, 94, -184, 730, 742, 162, -10, 708, 692, 773, 707, 855, 811, 703, 790, 871, 806, 9, 867, 840, 990, 1023, 1409, 194, 1397, 183, 1462, 178, -23, 1403, 247, 172, 1, -32, -170, 72, -508, -46, -365, -26, -146, 101, -18, -163, -422, -461, -146, -69, -78, -319, -334, -232, -99, 0, 47, -74, 0, -452, 14, -57, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, }; const int MATCH_PRED[] = { 0, 64, 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1038, 1053, 1067, 1082, 1096, 1111, 1125, 1139, 1154, 1168, 1183, 1197, 1211, 1226, 1240, 1255, 1269, 1284, 1298, 1312, 1327, 1341, 1356, 1370, 1385, 1399, 1413, 1428, 1442, 1457, 1471, 1486, 1500, 1514, 1529, 1543, 1558, 1572, 1586, 1601, 1615, 1630, 1644, 1659, 1673, 1687, 1702, 1716, 1731, 1745, 1760, 1774, 1788, 1803, 1817, 1832, 1846, 1861, 1875, 1889, 1904, 1918, 1933, 1947, 1961, 1976, 1990, 2005, 2019, 2034, 2047, }; template TPAQPredictor::TPAQPredictor(Context* ctx) : _sse0(256) , _sse1((T == true) ? 65536 : 256) { uint statesSize = 1 << 28; uint mixersSize = 1 << 12; uint hashSize = HASH_SIZE; uint extraMem = (T == true) ? 1 : 0; uint bufferSize = BUFFER_SIZE; uint bsVersion = 7; if (ctx != nullptr) { // Block size requested by the user // The user can request a big block size to force more states const int rbsz = ctx->getInt("blockSize", 32768); if (rbsz >= 64 * 1024 * 1024) statesSize = 1 << 28; else if (rbsz >= 16 * 1024 * 1024) statesSize = 1 << 27; else if (rbsz >= 4 * 1024 * 1024) statesSize = 1 << 26; else statesSize = (rbsz >= 1024 * 1024) ? 1 << 24 : 1 << 22; // Actual size of the current block // Too many mixers hurts compression for small blocks. // Too few mixers hurts compression for big blocks. const int absz = ctx->getInt("size", rbsz); if (absz >= 32 * 1024 * 1024) mixersSize = 1 << 16; else if (absz >= 16 * 1024 * 1024) mixersSize = 1 << 15; else if (absz >= 8 * 1024 * 1024) mixersSize = 1 << 14; else if (absz >= 4 * 1024 * 1024) mixersSize = 1 << 13; else mixersSize = (absz >= 1 * 1024 * 1024) ? 1 << 11 : 1 << 8; bufferSize = rbsz < BUFFER_SIZE ? rbsz : BUFFER_SIZE; const uint mxsz = absz < (1 << 26) ? absz * 16 : 1 << 30; hashSize = hashSize < mxsz ? hashSize : mxsz; bsVersion = ctx->getInt("bsVersion", bsVersion); } if (bsVersion > 6) { // The ring buffer and hash table use bit masks for indexing. // Normalize their sizes to powers of two before creating the masks. bufferSize = 1u << Global::_log2(bufferSize); hashSize = 1u << Global::_log2(hashSize); } mixersSize <<= (2 * extraMem); statesSize <<= (2 * extraMem); hashSize <<= (2 * extraMem); // Cap hash size for java compatibility if ((bsVersion > 5) && (hashSize > 1024 * 1024 * 1024)) hashSize = 1024 * 1024 * 1024; _statesMask = statesSize - 1; _mixersMask = (mixersSize - 1) & ~1; _hashMask = hashSize - 1; _bufferMask = bufferSize - 1; _mixers = new TPAQMixer[mixersSize]; _bigStatesMap = new uint8[statesSize]; _smallStatesMap0 = new uint8[1 << 16]; _smallStatesMap1 = new uint8[1 << 24]; _hashes = new int[hashSize]; _buffer = new byte[bufferSize]; reset(); } template bool TPAQPredictor::reset() { _pr = 2048; _c0 = 1; _c4 = 0; _c8 = 0; _pos = 0; _bpos = 8; _binCount = 0; _matchLen = 0; _matchPos = 0; _matchVal = 0; _hash = 0; _mixer = &_mixers[0]; memset(_bigStatesMap, 0, size_t(_statesMask + 1)); memset(_smallStatesMap0, 0, 1 << 16); memset(_smallStatesMap1, 0, 1 << 24); memset(_hashes, 0, sizeof(int) * size_t(_hashMask + 1)); memset(_buffer, 0, size_t(_bufferMask + 1)); _cp0 = &_smallStatesMap0[0]; _cp1 = &_smallStatesMap1[0]; _cp2 = &_bigStatesMap[0]; _cp3 = &_bigStatesMap[0]; _cp4 = &_bigStatesMap[0]; _cp5 = &_bigStatesMap[0]; _cp6 = &_bigStatesMap[0]; _ctx0 = _ctx1 = _ctx2 = _ctx3 = 0; _ctx4 = _ctx5 = _ctx6 = 0; return true; } template TPAQPredictor::~TPAQPredictor() { delete[] _bigStatesMap; delete[] _smallStatesMap0; delete[] _smallStatesMap1; delete[] _hashes; delete[] _buffer; delete[] _mixers; } // Update the probability model template void TPAQPredictor::update(int bit) { _mixer->update(bit); _c0 += (_c0 + bit); _bpos--; if (_bpos == 0) { _buffer[_pos & _bufferMask] = byte(_c0); _pos++; _c8 = (_c8 << 8) | ((_c4 >> 24) & 0xFF); _c4 = (_c4 << 8) | (_c0 & 0xFF); _hash = (((_hash * HASH) << 4) + _c4) & _hashMask; _c0 = 1; _bpos = 8; _binCount += ((_c4 >> 7) & 1); // Select Neural Net _mixer = &_mixers[(_c4 & _mixersMask) + (_matchLen != 0 ? 1 : 0)]; // Add contexts to NN _ctx0 = (_c4 & 0xFF) << 8; _ctx1 = (_c4 & 0xFFFF) << 8; _ctx2 = createContext(2, _c4 & 0x00FFFFFF); _ctx3 = createContext(3, _c4); if (_binCount < (_pos >> 2)) { // Mostly text or mixed _ctx4 = createContext(_ctx1, _c4 ^ (_c8 & 0xFFFF)); _ctx5 = (_c8 & MASK_F0F0F000) | ((_c4 & MASK_F0F0F000) >> 4); if (T == true) { const uint h1 = ((_c4 & MASK_80808080) == 0) ? _c4 & MASK_4F4FFFFF : _c4 & MASK_80808080; const uint h2 = ((_c8 & MASK_80808080) == 0) ? _c8 & MASK_4F4FFFFF : _c8 & MASK_80808080; _ctx6 = hash(h1 << 2, h2 >> 2); } } else { // Mostly binary _ctx4 = createContext(HASH + _matchLen, _c4 ^ (_c4 & 0x000FFFFF)); _ctx5 = _ctx0 | (_c8 << 16); if (T == true) { _ctx6 = hash(_c4 & 0xFFFF0000, _c8 >> 16); } } findMatch(); _matchVal = int(_buffer[_matchPos & _bufferMask]) | 0x100; // Keep track current position _hashes[_hash] = _pos; } // Get initial predictions // It has been observed that accessing memory via [ctx ^ c] is significantly faster // on SandyBridge/Windows and slower on SkyLake/Linux except when [ctx & 255 == 0] // (with c < 256). Hence, use XOR for _ctx5 which is the only context that fulfills // the condition. const int idx2 = (uint(_ctx2) + _c0) & _statesMask; const int idx3 = (uint(_ctx3) + _c0) & _statesMask; const int idx4 = (uint(_ctx4) + _c0) & _statesMask; const int idx5 = (uint(_ctx5) ^ _c0) & _statesMask; prefetchRead(&_bigStatesMap[idx2]); prefetchRead(&_bigStatesMap[idx3]); prefetchRead(&_bigStatesMap[idx4]); prefetchRead(&_bigStatesMap[idx5]); const uint8* table = STATE_TRANSITIONS[bit]; *_cp0 = table[*_cp0]; *_cp1 = table[*_cp1]; *_cp2 = table[*_cp2]; *_cp3 = table[*_cp3]; *_cp4 = table[*_cp4]; *_cp5 = table[*_cp5]; _cp0 = &_smallStatesMap0[_ctx0 + _c0]; const int p0 = STATE_MAP[*_cp0]; _cp1 = &_smallStatesMap1[_ctx1 + _c0]; const int p1 = STATE_MAP[*_cp1]; _cp2 = &_bigStatesMap[idx2]; const int p2 = STATE_MAP[*_cp2]; _cp3 = &_bigStatesMap[idx3]; const int p3 = STATE_MAP[*_cp3]; _cp4 = &_bigStatesMap[idx4]; const int p4 = STATE_MAP[*_cp4]; _cp5 = &_bigStatesMap[idx5]; const int p5 = STATE_MAP[*_cp5]; const int p7 = (_matchLen == 0) ? 0 : getMatchContextPred(); int p; if (T == false) { // Mix predictions using NN p = _mixer->get(p0, p1, p2, p3, p4, p5, p7, p7); // SSE (Secondary Symbol Estimation) if (_binCount < (_pos >> 3)) { p = (3 * _sse0.get(bit, p, _c0) + p) >> 2; } } else { // One more prediction const int idx6 = (uint(_ctx6) + _c0) & _statesMask; prefetchRead(&_bigStatesMap[idx6]); *_cp6 = table[*_cp6]; _cp6 = &_bigStatesMap[idx6]; const int p6 = STATE_MAP[*_cp6]; // Mix predictions using NN p = _mixer->get(p0, p1, p2, p3, p4, p5, p6, p7); // SSE (Secondary Symbol Estimation) if (_binCount < (_pos >> 3)) { p = _sse1.get(bit, p, _ctx0 + _c0); } else { if (_binCount >= (_pos >> 2)) p = (3 * _sse0.get(bit, p, _c0) + p) >> 2; p = (3 * _sse1.get(bit, p, _ctx0 + _c0) + p) >> 2; } } _pr = p + ((p < 2048) ? 1 : 0); } template void TPAQPredictor::findMatch() { // Update ongoing sequence match or detect match in the buffer (LZ like) if (_matchLen > 0) { if (_matchLen < MAX_LENGTH) _matchLen++; _matchPos++; return; } // Retrieve match position _matchPos = _hashes[_hash]; // Detect match if ((_matchPos != 0) && (uint(_pos - _matchPos) <= _bufferMask)) { int r = _matchLen + 2; // Compare four pairs at once when both logical ranges are physically // contiguous. Wrapping ranges use the scalar path below. while (r + 6 <= MAX_LENGTH) { const uint p0 = uint(_pos - r - 7) & _bufferMask; const uint p1 = uint(_matchPos - r - 7) & _bufferMask; if ((p0 > _bufferMask - 7) || (p1 > _bufferMask - 7)) break; // Read as big endian so the newest byte is in the low bits. // This makes trailingZeros() identify the first mismatch in // the same direction as the original pair-by-pair scan. const uint64 diff = uint64(BigEndian::readLong64(&_buffer[p0])) ^ uint64(BigEndian::readLong64(&_buffer[p1])); if (diff != 0) { // Round down to a complete pair. The scalar loop below // locates the exact mismatching pair. r += (Global::trailingZeros(diff) >> 4) << 1; break; } r += 8; } // Locate the first mismatching pair and preserve the original // even-length match semantics. while (r <= MAX_LENGTH) { if ((_buffer[(_pos - r - 1) & _bufferMask]) != (_buffer[(_matchPos - r - 1) & _bufferMask])) break; if ((_buffer[(_pos - r) & _bufferMask]) != (_buffer[(_matchPos - r) & _bufferMask])) break; r += 2; } _matchLen = r - 2; } } template inline int TPAQPredictor::hash(uint x, uint y) const { const int h = x * HASH ^ y * HASH; return (h >> 1) ^ (h >> 9) ^ (x >> 2) ^ (y >> 3) ^ HASH; } template inline int TPAQPredictor::createContext(uint ctxId, uint cx) const { cx = cx * 987654323 + ctxId; cx = (cx << 16) | (cx >> 16); return cx * 123456791 + ctxId; } // Get a prediction from the match model in [-2047..2048] template inline int TPAQPredictor::getMatchContextPred() { const uint matchPrefix = uint(_matchVal) >> _bpos; if (_c0 == matchPrefix) { return (((_matchVal >> (_bpos - 1)) & 1) != 0) ? MATCH_PRED[_matchLen - 1] : -MATCH_PRED[_matchLen - 1]; } _matchLen = 0; return 0; } } #endif kanzi-cpp-2.6.0/src/io/000077500000000000000000000000001525576310500146225ustar00rootroot00000000000000kanzi-cpp-2.6.0/src/io/CompressedInputStream.cpp000066400000000000000000001262761525576310500216440ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include "CompressedInputStream.hpp" #include "IOException.hpp" #include "../Error.hpp" #include "../entropy/EntropyDecoderFactory.hpp" #include "../transform/TransformFactory.hpp" #include "../util/fixedbuf.hpp" using namespace kanzi; using namespace std; // Clear ownership before delete so a throwing destructor cannot leave a // dangling non-null pointer for later cleanup. #define DELETE_AND_NULL(TYPE, PTR) \ do { \ TYPE* knzTmpPtr = (PTR); \ (PTR) = nullptr; \ delete knzTmpPtr; \ } while (0) const int CompressedInputStream::BITSTREAM_TYPE = 0x4B414E5A; // "KANZ" const int CompressedInputStream::BITSTREAM_FORMAT_VERSION = 7; const int CompressedInputStream::DEFAULT_BUFFER_SIZE = 256 * 1024; const int CompressedInputStream::EXTRA_BUFFER_SIZE = 512; const kanzi::byte CompressedInputStream::COPY_BLOCK_MASK = kanzi::byte(0x80); const kanzi::byte CompressedInputStream::TRANSFORMS_MASK = kanzi::byte(0x10); const int CompressedInputStream::MIN_BITSTREAM_BLOCK_SIZE = 1024; const int CompressedInputStream::MAX_BITSTREAM_BLOCK_SIZE = 1024 * 1024 * 1024; const int CompressedInputStream::TRANSFORMED_COPY_VERSION = 7; const int CompressedInputStream::CANCEL_TASKS_ID = -1; const int CompressedInputStream::MAX_CONCURRENCY = 64; const int CompressedInputStream::MAX_BLOCK_ID = int((uint(1) << 31) - 1); CompressedInputStream::CompressedInputStream(InputStream& is, int tasks, const string& entropy, const string& transform, int blockSize, int checksum, uint64 originalSize, #ifdef CONCURRENCY_ENABLED ThreadPool* pool, #endif bool headerless, int bsVersion) : InputStream(is.rdbuf()) , _parentCtx(nullptr) { #ifdef CONCURRENCY_ENABLED if ((tasks <= 0) || (tasks > MAX_CONCURRENCY)) { stringstream ss; ss << "The number of jobs must be in [1.." << MAX_CONCURRENCY << "], got " << tasks; throw invalid_argument(ss.str()); } _pool = pool; // may be null #else if (tasks != 1) throw invalid_argument("The number of jobs is limited to 1 in this version"); #endif _hasher32 = nullptr; _hasher64 = nullptr; _blockId = 0; _bufferId = 0; _maxBufferId = 0; _submitBlockId = 0; _blockSize = blockSize; _bufferThreshold = 0; _available = 0; _entropyType = EntropyDecoderFactory::getType(entropy.c_str()); // throws on error _transformType = TransformFactory::getType(transform.c_str()); // throws on error _initialized = 0; _closed = 0; _gcount = 0; _ibs = new DefaultInputBitStream(is, DEFAULT_BUFFER_SIZE); _jobs = tasks; _outputSize = originalSize; _nbInputBlocks = 0; _buffers = new SliceArray*[2 * _jobs]; _headless = headerless; _consumeBlockId = 0; if (_headless == true) { if ((_blockSize < MIN_BITSTREAM_BLOCK_SIZE) || (_blockSize > MAX_BITSTREAM_BLOCK_SIZE)) { stringstream ss; ss << "Invalid or missing block size: " << _blockSize; throw invalid_argument(ss.str()); } _ctx.putInt("bsVersion", bsVersion); _ctx.putString("entropy", entropy); _ctx.putString("transform", transform); _ctx.putInt("blockSize", blockSize); if (checksum == 32) { _hasher32 = new XXHash32(BITSTREAM_TYPE); _hasher64 = nullptr; } else if (checksum == 64) { _hasher32 = nullptr; _hasher64 = new XXHash64(BITSTREAM_TYPE); } else if (checksum != 0) { throw invalid_argument("The block checksum size must be 0, 32 or 64"); } } _jobsPerTask.resize(_jobs); std::fill(_jobsPerTask.begin(), _jobsPerTask.end(), 1); #ifdef CONCURRENCY_ENABLED _futures.resize(_jobs); #else _results.resize(_jobs); #endif for (int i = 0; i < 2 * _jobs; i++) _buffers[i] = new SliceArray(nullptr, 0, 0); } CompressedInputStream::CompressedInputStream(InputStream& is, Context& ctx, bool headerless) : InputStream(is.rdbuf()) , _ctx(ctx) , _parentCtx(&ctx) { int tasks = _ctx.getInt("jobs", 1); #ifdef CONCURRENCY_ENABLED if ((tasks <= 0) || (tasks > MAX_CONCURRENCY)) { stringstream ss; ss << "The number of jobs must be in [1.." << MAX_CONCURRENCY << "], got " << tasks; throw invalid_argument(ss.str()); } _pool = _ctx.getPool(); // may be null #else if (tasks != 1) throw invalid_argument("The number of jobs is limited to 1 in this version"); #endif _blockId = 0; _bufferId = 0; _maxBufferId = 0; _submitBlockId = 0; _blockSize = 0; _bufferThreshold = 0; _available = 0; _entropyType = EntropyDecoderFactory::NONE_TYPE; _transformType = TransformFactory::NONE_TYPE; _initialized = 0; _closed = 0; _gcount = 0; _ibs = new DefaultInputBitStream(is, DEFAULT_BUFFER_SIZE); _jobs = tasks; _hasher32 = nullptr; _hasher64 = nullptr; _outputSize = 0; _nbInputBlocks = 0; _headless = headerless; _consumeBlockId = 0; if (_headless == true) { // Validation of required values // Optional bsVersion const int bsVersion = _ctx.getInt("bsVersion", BITSTREAM_FORMAT_VERSION); if (bsVersion > BITSTREAM_FORMAT_VERSION) { stringstream ss; ss << "Invalid or missing bitstream version, cannot read this version of the stream: " << bsVersion; throw invalid_argument(ss.str()); } _ctx.putInt("bsVersion", bsVersion); string entropy = _ctx.getString("entropy"); _entropyType = EntropyDecoderFactory::getType(entropy.c_str()); // throws on error string transform = _ctx.getString("transform"); _transformType = TransformFactory::getType(transform.c_str()); // throws on error _blockSize = _ctx.getInt("blockSize", 0); if ((_blockSize < MIN_BITSTREAM_BLOCK_SIZE) || (_blockSize > MAX_BITSTREAM_BLOCK_SIZE)) { stringstream ss; ss << "Invalid or missing block size: " << _blockSize; throw invalid_argument(ss.str()); } _bufferThreshold = _blockSize; // Optional outputSize if (_ctx.has("outputSize")) { _outputSize = _ctx.getLong("outputSize", 0); if ((_outputSize < 0) || (_outputSize >= (int64(1) << 48))) _outputSize = 0; // not provided } const int nbBlocks = int((_outputSize + int64(_blockSize - 1)) / int64(_blockSize)); _nbInputBlocks = min(nbBlocks, MAX_CONCURRENCY - 1); // Optional checksum int checksum = ctx.getInt("checksum", 0); if (checksum == 0) { _hasher32 = nullptr; _hasher64 = nullptr; } else if (checksum == 32) { _hasher32 = new XXHash32(BITSTREAM_TYPE); _hasher64 = nullptr; } else if (checksum == 64) { _hasher32 = nullptr; _hasher64 = new XXHash64(BITSTREAM_TYPE); } else { throw invalid_argument("The block checksum size must be 0, 32 or 64"); } } _jobsPerTask.resize(_jobs); std::fill(_jobsPerTask.begin(), _jobsPerTask.end(), 1); #ifdef CONCURRENCY_ENABLED _futures.resize(_jobs); #else _results.resize(_jobs); #endif _buffers = new SliceArray*[2 * _jobs]; for (int i = 0; i < 2 * _jobs; i++) _buffers[i] = new SliceArray(nullptr, 0, 0); } CompressedInputStream::~CompressedInputStream() { try { close(); } catch (const exception&) { // Ignore and continue } for (int i = 0; i < 2 * _jobs; i++) { if (_buffers[i]->_array != nullptr) delete[] _buffers[i]->_array; delete _buffers[i]; } delete[] _buffers; delete _ibs; if (_hasher32 != nullptr) { delete _hasher32; _hasher32 = nullptr; } if (_hasher64 != nullptr) { delete _hasher64; _hasher64 = nullptr; } } void CompressedInputStream::submitBlock(int bufferId) { const int blkSize = max(_blockSize + EXTRA_BUFFER_SIZE, _blockSize + (_blockSize >> 4)); if (_buffers[bufferId]->_length < blkSize) { if (_buffers[bufferId]->_array != nullptr) delete[] _buffers[bufferId]->_array; _buffers[bufferId]->_array = new kanzi::byte[blkSize]; _buffers[bufferId]->_length = blkSize; } Context copyCtx(_ctx); copyCtx.putLong("tType", _transformType); copyCtx.putInt("eType", _entropyType); copyCtx.putInt("blockId", _submitBlockId + 1); copyCtx.putInt("jobs", _jobsPerTask[bufferId]); copyCtx.putInt("tasks", _jobs); _buffers[bufferId]->_index = 0; _buffers[_jobs + bufferId]->_index = 0; DecodingTask* task = new DecodingTask( _buffers[bufferId], _buffers[_jobs + bufferId], blkSize, _ibs, _hasher32, _hasher64, #ifdef CONCURRENCY_ENABLED &_blockMutex, &_blockCondition, #endif &_blockId, _listeners, copyCtx); #ifdef CONCURRENCY_ENABLED std::shared_ptr> safeTask(task); auto taskRunner = [safeTask]() { return safeTask->run(); }; if (_pool == nullptr) { // std::async returns std::future _futures[bufferId] = std::async(std::launch::async, taskRunner); } else { // pool->schedule returns std::future _futures[bufferId] = _pool->schedule(taskRunner); } #else // Synchronous execution try { _results[bufferId] = task->run(); delete task; } catch (...) { delete task; throw; } #endif _submitBlockId++; } int CompressedInputStream::_get(int inc) { try { if (LOAD_ATOMIC(_initialized) == 0) { readHeader(); for (int i = 0; i < _jobs; i++) submitBlock(i); } if (_available == 0) { if (LOAD_ATOMIC(_closed) == 1) throw ios_base::failure("Stream closed"); DecodingTaskResult res; #ifdef CONCURRENCY_ENABLED if (_futures[_bufferId].valid()) { res = _futures[_bufferId].get(); } else { setstate(ios::eofbit); return EOF; } #else res = _results[_bufferId]; #endif if (res._error != 0) throw IOException(res._msg, res._error); if (res._decoded > _blockSize) { stringstream ss; ss << "Block " << res._blockId << " incorrectly decompressed"; throw IOException(ss.str(), Error::ERR_PROCESS_BLOCK); } // Fire events if (!_listeners.empty()) { Event::HashType hashType = Event::NO_HASH; if (_hasher32 != nullptr) hashType = Event::SIZE_32; else if (_hasher64 != nullptr) hashType = Event::SIZE_64; Event evt(Event::AFTER_TRANSFORM, res._blockId, int64(res._decoded), res._completionTime, res._checksum, hashType); CompressedInputStream::notifyListeners(_listeners, evt); } _available = res._decoded; if (_available == 0) { if (res._skipped == false) { setstate(ios::eofbit); return EOF; } submitBlock(_bufferId); _bufferId = (_bufferId + 1) % _jobs; _consumeBlockId++; } _buffers[_bufferId]->_index = 0; } int res = int(_buffers[_bufferId]->_array[_buffers[_bufferId]->_index]); if (inc == 0) return res; _available -= inc; _buffers[_bufferId]->_index += inc; if (_available == 0) { submitBlock(_bufferId); _bufferId = (_bufferId + 1) % _jobs; _consumeBlockId++; } return res; } catch (const IOException&) { setstate(ios::badbit); throw; } catch (const exception&) { setstate(ios::badbit); throw; } catch (...) { setstate(ios::badbit); throw; } } istream& CompressedInputStream::read(char* data, streamsize length) { if (length < 0) throw ios_base::failure("Invalid buffer size"); try { streamsize remaining = length; _gcount = 0; while (remaining > 0) { // Reuse _get(0) logic implicitly if (LOAD_ATOMIC(_initialized) == 0) { readHeader(); for (int i = 0; i < _jobs; i++) submitBlock(i); } if (_available == 0) { DecodingTaskResult res; #ifdef CONCURRENCY_ENABLED if (_futures[_bufferId].valid()) { res = _futures[_bufferId].get(); } else { setstate(ios::eofbit); break; } #else res = _results[_bufferId]; #endif if (res._error != 0) throw IOException(res._msg, res._error); if (res._decoded > _blockSize) { stringstream ss; ss << "Block " << res._blockId << " incorrectly decompressed"; throw IOException(ss.str(), Error::ERR_PROCESS_BLOCK); } if (!_listeners.empty()) { Event::HashType hashType = Event::NO_HASH; if (_hasher32 != nullptr) hashType = Event::SIZE_32; else if (_hasher64 != nullptr) hashType = Event::SIZE_64; Event evt(Event::AFTER_TRANSFORM, res._blockId, int64(res._decoded), res._completionTime, res._checksum, hashType); CompressedInputStream::notifyListeners(_listeners, evt); } _available = res._decoded; _buffers[_bufferId]->_index = 0; if ((_available == 0) && (res._skipped == false)) { setstate(ios::eofbit); break; } } const streamsize lenChunk = min(remaining, streamsize(_available)); if (lenChunk > 0) { memcpy(&data[_gcount], &_buffers[_bufferId]->_array[_buffers[_bufferId]->_index], size_t(lenChunk)); _buffers[_bufferId]->_index += int(lenChunk); _gcount += lenChunk; remaining -= lenChunk; _available -= lenChunk; } if (_available == 0) { submitBlock(_bufferId); _bufferId = (_bufferId + 1) % _jobs; _consumeBlockId++; } } return *this; } catch (const IOException&) { setstate(ios::badbit); throw; } catch (const exception&) { setstate(ios::badbit); throw; } catch (...) { setstate(ios::badbit); throw; } } void CompressedInputStream::readHeader() { if (EXCHANGE_ATOMIC(_initialized, 1) == 1) return; if (_headless == true) return; // Read stream type const int type = int(_ibs->readBits(32)); // Sanity check if (type != BITSTREAM_TYPE) { throw IOException("Invalid stream type", Error::ERR_INVALID_FILE); } // Read stream version const int bsVersion = int(_ibs->readBits(4)); // Sanity check if (bsVersion > BITSTREAM_FORMAT_VERSION) { stringstream ss; ss << "Invalid bitstream, cannot read this version of the stream: " << bsVersion; throw IOException(ss.str(), Error::ERR_STREAM_VERSION); } _ctx.putInt("bsVersion", bsVersion); uint64 ckSize = 0; // Read block checksum if (bsVersion >= 6) { ckSize = _ibs->readBits(2); if (ckSize == 1) { _hasher32 = new XXHash32(BITSTREAM_TYPE); } else if (ckSize == 2) { _hasher64 = new XXHash64(BITSTREAM_TYPE); } else if (ckSize == 3) { throw IOException("Invalid bitstream, incorrect block checksum size", Error::ERR_INVALID_FILE); } } else { if (_ibs->readBit() == 1) _hasher32 = new XXHash32(BITSTREAM_TYPE); } try { // Read entropy codec _entropyType = short(_ibs->readBits(5)); _ctx.putString("entropy", EntropyDecoderFactory::getName(_entropyType)); } catch (const invalid_argument&) { stringstream err; err << "Invalid bitstream, unknown entropy type: " << _entropyType; throw IOException(err.str(), Error::ERR_INVALID_CODEC); } try { // Read transform: 8*6 bits _transformType = _ibs->readBits(48); _ctx.putString("transform", TransformFactory::getName(_transformType)); } catch (const invalid_argument&) { stringstream err; err << "Invalid bitstream, unknown transform type: " << _transformType; throw IOException(err.str(), Error::ERR_INVALID_CODEC); } // Read block size _blockSize = int(_ibs->readBits(28) << 4); _ctx.putInt("blockSize", _blockSize); _bufferThreshold = _blockSize; if ((_blockSize < MIN_BITSTREAM_BLOCK_SIZE) || (_blockSize > MAX_BITSTREAM_BLOCK_SIZE)) { stringstream ss; ss << "Invalid bitstream, incorrect block size: " << _blockSize; throw IOException(ss.str(), Error::ERR_BLOCK_SIZE); } // Read original size // 0 -> not provided, <2^16 -> 1, <2^32 -> 2, <2^48 -> 3 const int szMask = int(_ibs->readBits(2)); if (szMask != 0) { _outputSize = _ibs->readBits(16 * szMask); if (_parentCtx != nullptr) _parentCtx->putLong("outputSize", _outputSize); const int nbBlocks = int((_outputSize + int64(_blockSize - 1)) / int64(_blockSize)); _nbInputBlocks = min(nbBlocks, MAX_CONCURRENCY - 1); } if (bsVersion >= 6) { // Padding _ibs->readBits(15); } // Assign optimal number of tasks and jobs per task (if the number of blocks is available) if (_jobs > 1) { // Limit the number of tasks if there are fewer blocks that _jobs int nbTasks = (_nbInputBlocks != 0) ? min(_nbInputBlocks, _jobs) : _jobs; Global::computeJobsPerTask(&_jobsPerTask[0], _jobs, nbTasks); } else { _jobsPerTask[0] = 1; } // Read & verify checksum const int crcSize = bsVersion <= 5 ? 16 : 24; const uint32 cksum1 = uint32(_ibs->readBits(crcSize)); uint32 seed = (bsVersion >= 6 ? 0x01030507 : 1) * uint32(bsVersion); const uint32 HASH = 0x1E35A7BD; uint32 cksum2 = HASH * seed; typedef uint32 (*Mix32Function)(uint32, uint32, uint32); const Mix32Function mix32 = (bsVersion >= 7) ? CompressedInputStream::mix32_v7 : CompressedInputStream::mix32_v6; if (bsVersion >= 6) cksum2 = mix32(cksum2, HASH, uint32(ckSize)); cksum2 = mix32(cksum2, HASH, uint32(_entropyType)); cksum2 = mix32(cksum2, HASH, uint32(_transformType >> 32)); cksum2 = mix32(cksum2, HASH, uint32(_transformType)); cksum2 = mix32(cksum2, HASH, uint32(_blockSize)); if (szMask != 0) { cksum2 = mix32(cksum2, HASH, uint32(_outputSize >> 32)); cksum2 = mix32(cksum2, HASH, uint32(_outputSize)); } cksum2 = (cksum2 >> 23) ^ (cksum2 >> 3); if (cksum1 != (cksum2 & ((1 << crcSize) - 1))) throw IOException("Invalid bitstream, header checksum mismatch", Error::ERR_CRC_CHECK); if (_listeners.size() > 0) { Event::HeaderInfo info; info.inputName = _ctx.getString("inputName", ""); info.bsVersion = bsVersion; info.checksumSize = int(32 * ckSize); info.blockSize = _blockSize; info.entropyType = EntropyDecoderFactory::getName(_entropyType); info.transformType = TransformFactory::getName(_transformType); int64 fileSize = _ctx.getLong("fileSize", 0); info.fileSize = (fileSize >= 0) ? fileSize : -1; info.originalSize = (szMask != 0) ? _outputSize : -1; WallTimer timer; Event evt(Event::AFTER_HEADER_DECODING, 0, info, timer.getCurrentTime()); notifyListeners(_listeners, evt); } } bool CompressedInputStream::addListener(Listener& bl) { _listeners.push_back(&bl); return true; } bool CompressedInputStream::removeListener(Listener& bl) { std::vector*>::iterator it = find(_listeners.begin(), _listeners.end(), &bl); if (it == _listeners.end()) return false; _listeners.erase(it); return true; } void CompressedInputStream::close() { if (EXCHANGE_ATOMIC(_closed, 1) == 1) return; // Signal to break the waits in DecodingTask::run immediately and // ensure no thread is writing to _buffers before we delete them. #ifdef CONCURRENCY_ENABLED { std::lock_guard lock(_blockMutex); STORE_ATOMIC(_blockId, CANCEL_TASKS_ID); } _blockCondition.notify_all(); for (size_t i = 0; i < _futures.size(); i++) { if (_futures[i].valid()) { try { _futures[i].get(); } catch (...) { // Ignore exceptions, we are closing anyway. } } } #else STORE_ATOMIC(_blockId, CANCEL_TASKS_ID); #endif try { _ibs->close(); } catch (const BitStreamException& e) { throw IOException(e.what(), e.error()); } _available = 0; // Force subsequent reads to trigger submitBlock immediately _bufferThreshold = 0; // Buffer cleanup: force error on any subsequent read attempt for (int i = 0; i < 2 * _jobs; i++) { if (_buffers[i]->_array != nullptr) delete[] _buffers[i]->_array; _buffers[i]->_array = nullptr; _buffers[i]->_length = 0; _buffers[i]->_index = 0; } } void CompressedInputStream::notifyListeners(vector*>& listeners, const Event& evt) { for (vector*>::iterator it = listeners.begin(); it != listeners.end(); ++it) (*it)->processEvent(evt); } template DecodingTask::DecodingTask(SliceArray* iBuffer, SliceArray* oBuffer, int blockSize, DefaultInputBitStream* ibs, XXHash32* hasher32, XXHash64* hasher64, #ifdef CONCURRENCY_ENABLED std::mutex* blockMutex, std::condition_variable* blockCondition, #endif atomic_int_t* processedBlockId, vector*>& listeners, const Context& ctx) : _listeners(listeners) , _ctx(ctx) { _blockLength = blockSize; _data = iBuffer; _buffer = oBuffer; _ibs = ibs; _hasher32 = hasher32; _hasher64 = hasher64; #ifdef CONCURRENCY_ENABLED _blockMutex = blockMutex; _blockCondition = blockCondition; #endif _processedBlockId = processedBlockId; } template void DecodingTask::storeProcessedBlockId(int value) { #ifdef CONCURRENCY_ENABLED { std::lock_guard lock(*_blockMutex); STORE_ATOMIC(*_processedBlockId, value); } _blockCondition->notify_all(); #else STORE_ATOMIC(*_processedBlockId, value); #endif } template int DecodingTask::readBlockHeader(InputBitStream* ibs, int bsVersion, uint64 encodedBlockLength, uint64 transformType, ParsedBlockHeader& header, string& msg) { if (encodedBlockLength < 8) { msg = "Invalid block size"; return Error::ERR_BLOCK_SIZE; } const kanzi::byte mode = kanzi::byte(ibs->readBits(8)); kanzi::byte skipFlags = kanzi::byte(0); bool hasSkipFlags = false; const bool copyBlock = (mode & CompressedInputStream::COPY_BLOCK_MASK) != kanzi::byte(0); if (copyBlock == true) { if ((bsVersion >= CompressedInputStream::TRANSFORMED_COPY_VERSION) && ((mode & CompressedInputStream::TRANSFORMS_MASK) != kanzi::byte(0))) { header._transformedCopy = true; if (TransformFactory::getTransformCount(transformType) > 4) hasSkipFlags = true; else skipFlags = (mode << 4) | kanzi::byte(0x0F); } } else if ((mode & CompressedInputStream::TRANSFORMS_MASK) != kanzi::byte(0)) { hasSkipFlags = true; } else { skipFlags = (mode << 4) | kanzi::byte(0x0F); } const int dataSize = 1 + (int(mode >> 5) & 0x03); header._size = uint(1 + dataSize + (hasSkipFlags ? 1 : 0) + (bsVersion >= 7 ? 1 : 0)); if (encodedBlockLength < uint64(header._size << 3)) { msg = "Invalid block size"; return Error::ERR_BLOCK_SIZE; } header._bytes[0] = mode; uint idx = 1; if (hasSkipFlags == true) { skipFlags = kanzi::byte(ibs->readBits(8)); header._bytes[idx++] = skipFlags; } uint32 preTransformLength = 0; for (int i = 0; i < dataSize; i++) { const kanzi::byte value = kanzi::byte(ibs->readBits(8)); header._bytes[idx++] = value; preTransformLength = (preTransformLength << 8) | uint32(uint8(value)); } header._skipFlags = skipFlags; header._preTransformLength = int(preTransformLength); header._rawCopy = copyBlock && !header._transformedCopy; if (bsVersion >= 7) { const uint32 headerChecksum = uint32(ibs->readBits(8)); header._bytes[idx] = kanzi::byte(headerChecksum); const uint32 hash = 0x1E35A7BDu; uint32 cksum = hash * 0x01030507u; cksum = CompressedInputStream::mix32_v7(cksum, hash, uint32(uint8(mode))); cksum = CompressedInputStream::mix32_v7(cksum, hash, uint32(uint8(skipFlags))); cksum = CompressedInputStream::mix32_v7(cksum, hash, preTransformLength); cksum = CompressedInputStream::mix32_v7(cksum, hash, uint32(encodedBlockLength >> 32)); cksum = CompressedInputStream::mix32_v7(cksum, hash, uint32(encodedBlockLength)); cksum = (cksum >> 23) ^ (cksum >> 3); if (headerChecksum != (cksum & 0xFFu)) { msg = "Invalid bitstream, block header checksum mismatch"; return Error::ERR_CRC_CHECK; } } return 0; } // Decode mode + transformed entropy coded data // mode | 0b1yy0xxxx => raw copy block // | 0b1yy1xxxx => transformed copy block (version >= 7) // | 0b0yy00000 => size(size(block))-1 // case 4 transforms or less // | 0b0001xxxx => transform sequence skip flags (1 means skip) // case more than 4 transforms // | 0b0yy00000 0bxxxxxxxx => transform sequence skip flags in next kanzi::byte (1 means skip) template T DecodingTask::run() { int blockId = _ctx.getInt("blockId"); bool streamPerTask = _ctx.getInt("tasks") > 1; const int bsVersion = _ctx.getInt("bsVersion", CompressedInputStream::BITSTREAM_FORMAT_VERSION); uint64 tType = _ctx.getLong("tType"); short eType = short(_ctx.getInt("eType")); #ifdef CONCURRENCY_ENABLED { std::unique_lock lock(*_blockMutex); _blockCondition->wait(lock, [this, blockId]() { const int taskId = LOAD_ATOMIC(*_processedBlockId); return (taskId == CompressedInputStream::CANCEL_TASKS_ID) || (taskId == blockId - 1); }); } if (LOAD_ATOMIC(*_processedBlockId) == CompressedInputStream::CANCEL_TASKS_ID) { // Skip, an error occurred return T(*_data, blockId, 0, 0, 0, "Canceled"); } #endif uint64 checksum1 = 0; EntropyDecoder* ed = nullptr; InputBitStream* ibs = nullptr; TransformSequence* transform = nullptr; try { // Read shared bitstream sequentially (each task is gated by _processedBlockId) #if !defined(_MSC_VER) || _MSC_VER > 1500 const uint64 blockOffset = _ibs->tell(); #endif const uint lr = 3 + uint(_ibs->readBits(5)); uint64 read = _ibs->readBits(lr); if (read == 0) { storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); return T(*_data, blockId, 0, 0, 0, "Success"); } const uint64 encodedBlockBytes = (read + 7) >> 3; const uint64 encodedBlockLength = read; const int from = _ctx.getInt("from", 1); const int to = _ctx.getInt("to", CompressedInputStream::MAX_BLOCK_ID); const bool bufferBlock = (streamPerTask == true) || (blockId < from); const uint maxTransformedCopySize = uint(max(_blockLength + _blockLength / 2, 2048u)); // Keep the normal bitstream cap for entropy-coded blocks, while // allowing transformed-copy blocks to carry bounded transform expansion. const uint maxEntropyTransformSize = uint(min(maxTransformedCopySize, uint(CompressedInputStream::MAX_BITSTREAM_BLOCK_SIZE))); ParsedBlockHeader blockHeader; bool blockHeaderParsed = false; if ((bsVersion >= 7) && (bufferBlock == true)) { string msg; const int error = readBlockHeader(_ibs, bsVersion, encodedBlockLength, tType, blockHeader, msg); if (error != 0) { storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); return T(*_data, blockId, 0, 0, error, msg); } blockHeaderParsed = true; const uint maxAllowedTransformSize = blockHeader._transformedCopy ? maxTransformedCopySize : maxEntropyTransformSize; if ((blockHeader._preTransformLength <= 0) || (uint(blockHeader._preTransformLength) > maxAllowedTransformSize)) { storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); stringstream ss; ss << "Invalid compressed block length: " << blockHeader._preTransformLength; return T(*_data, blockId, 0, 0, Error::ERR_READ_FILE, ss.str()); } const uint64 checksumSize = (_hasher64 != nullptr) ? 8 : ((_hasher32 != nullptr) ? 4 : 0); const uint64 maxEncodedBlockBytes = uint64(blockHeader._preTransformLength) + uint64(blockHeader._size) + checksumSize; if (encodedBlockBytes > maxEncodedBlockBytes) { storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); return T(*_data, blockId, 0, 0, Error::ERR_BLOCK_SIZE, "Invalid block size"); } read -= (uint64(blockHeader._size) << 3); } else if (encodedBlockBytes > uint64((numeric_limits::max)())) { storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); return T(*_data, blockId, 0, 0, Error::ERR_BLOCK_SIZE, "Invalid block size"); } const uint r = uint(encodedBlockBytes); // Read from the shared bitstream if // - there is one that one task (each with their own local bitstream) // - the block is going to be skipped (bits must be consumed) if (bufferBlock == true) { if (_data->_length < int(max(_blockLength, r))) { _data->_length = int(max(_blockLength, r)); delete[] _data->_array; _data->_array = new kanzi::byte[_data->_length]; } for (uint i = 0; i < blockHeader._size; i++) _data->_array[i] = blockHeader._bytes[i]; for (int n = int(blockHeader._size); read > 0; ) { const uint chkSize = uint(min(read, uint64(1) << 30)); _ibs->readBits(&_data->_array[n], chkSize); n += ((chkSize + 7) >> 3); read -= uint64(chkSize); } } // After completion of the bitstream reading, increment the block id. // It unblocks the task processing the next block (if any) storeProcessedBlockId(blockId); // Check if the block must be skipped if (blockId < from) { return T(*_data, blockId, 0, 0, 0, "Skipped", true); } else if (blockId >= to) { return T(*_data, blockId, 0, 0, 0, "Success"); } ifixedbuf buf(reinterpret_cast(&_data->_array[0]), streamsize(r)); istream ios(&buf); ibs = (streamPerTask == true) ? new DefaultInputBitStream(ios) : _ibs; if (blockHeaderParsed == false) { string msg; const int error = readBlockHeader(ibs, bsVersion, encodedBlockLength, tType, blockHeader, msg); if (error != 0) { storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); if (streamPerTask == true) { try { DELETE_AND_NULL(InputBitStream, ibs); } catch (...) { } } return T(*_data, blockId, 0, checksum1, error, msg); } } else { ibs->readBits(blockHeader._size << 3); } const kanzi::byte skipFlags = blockHeader._skipFlags; const bool transformedCopy = blockHeader._transformedCopy; const int preTransformLength = blockHeader._preTransformLength; if (blockHeader._rawCopy == true) { tType = TransformFactory::NONE_TYPE; eType = EntropyDecoderFactory::NONE_TYPE; } else if (transformedCopy == true) { eType = EntropyDecoderFactory::NONE_TYPE; } const uint maxAllowedTransformSize = transformedCopy ? maxTransformedCopySize : maxEntropyTransformSize; if ((preTransformLength <= 0) || (uint(preTransformLength) > maxAllowedTransformSize)) { // Error => cancel concurrent decoding tasks storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); stringstream ss; ss << "Invalid compressed block length: " << preTransformLength; if (streamPerTask == true) { try { DELETE_AND_NULL(InputBitStream, ibs); } catch (...) { } } return T(*_data, blockId, 0, checksum1, Error::ERR_READ_FILE, ss.str()); } if ((bsVersion >= 7) && (blockHeaderParsed == false)) { const uint64 checksumSize = (_hasher64 != nullptr) ? 8 : ((_hasher32 != nullptr) ? 4 : 0); const uint64 maxEncodedBlockBytes = uint64(preTransformLength) + uint64(blockHeader._size) + checksumSize; if (encodedBlockBytes > maxEncodedBlockBytes) { storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); if (streamPerTask == true) { try { DELETE_AND_NULL(InputBitStream, ibs); } catch (...) { } } return T(*_data, blockId, 0, checksum1, Error::ERR_BLOCK_SIZE, "Invalid block size"); } } Event::HashType hashType = Event::NO_HASH; WallTimer timer; // Extract checksum from bitstream (if any) if (_hasher32 != nullptr) { checksum1 = ibs->readBits(32); hashType = Event::SIZE_32; } else if (_hasher64 != nullptr) { checksum1 = ibs->readBits(64); hashType = Event::SIZE_64; } if (_listeners.size() > 0) { #if !defined(_MSC_VER) || _MSC_VER > 1500 if (_ctx.getInt("verbosity", 0) > 4) { Event evt1(Event::BLOCK_INFO, blockId, int64(r), timer.getCurrentTime(), checksum1, hashType, blockOffset, uint8(skipFlags)); CompressedInputStream::notifyListeners(_listeners, evt1); } #endif // Notify before entropy Event evt2(Event::BEFORE_ENTROPY, blockId, int64(r), timer.getCurrentTime(), checksum1, hashType); CompressedInputStream::notifyListeners(_listeners, evt2); } const int bufferSize = max(int(_blockLength), preTransformLength + CompressedInputStream::EXTRA_BUFFER_SIZE); if (_buffer->_length < bufferSize) { _buffer->_length = bufferSize; if (_buffer->_array != nullptr) delete[] _buffer->_array; _buffer->_array = new kanzi::byte[_buffer->_length]; } const int savedIdx = _data->_index; _ctx.putInt("size", preTransformLength); if (transformedCopy == true) { // The encoder kept the transformed bytes verbatim because entropy // coding expanded the block. Read the payload directly into the // post-entropy buffer, then continue with the inverse transform. for (uint n = 0, remaining = uint(preTransformLength); remaining > 0; ) { const uint chunk = min(remaining, uint(1) << 27); if (ibs->readBits(&_buffer->_array[n], 8 * chunk) != 8 * chunk) { storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); if (streamPerTask == true) { try { DELETE_AND_NULL(InputBitStream, ibs); } catch (...) { } } return T(*_data, blockId, 0, checksum1, Error::ERR_PROCESS_BLOCK, "Entropy decoding failed"); } n += chunk; remaining -= chunk; } if (streamPerTask == true) { DELETE_AND_NULL(InputBitStream, ibs); } } else { // Each block is decoded separately // Rebuild the entropy decoder to reset block statistics ed = EntropyDecoderFactory::newDecoder(*ibs, _ctx, eType); // Block entropy decode if (ed->decode(_buffer->_array, 0, preTransformLength) != preTransformLength) { // Error => cancel concurrent decoding tasks storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); try { DELETE_AND_NULL(EntropyDecoder, ed); } catch (...) { } if (streamPerTask == true) { try { DELETE_AND_NULL(InputBitStream, ibs); } catch (...) { } } return T(*_data, blockId, 0, checksum1, Error::ERR_PROCESS_BLOCK, "Entropy decoding failed"); } if (streamPerTask == true) { DELETE_AND_NULL(InputBitStream, ibs); } DELETE_AND_NULL(EntropyDecoder, ed); } if (_listeners.size() > 0) { // Notify after entropy Event evt1(Event::AFTER_ENTROPY, blockId, int64(preTransformLength), timer.getCurrentTime(), checksum1, hashType); CompressedInputStream::notifyListeners(_listeners, evt1); // Notify before transform (block size after entropy decoding) Event evt2(Event::BEFORE_TRANSFORM, blockId, int64(preTransformLength), timer.getCurrentTime(), checksum1, hashType); CompressedInputStream::notifyListeners(_listeners, evt2); } transform = TransformFactory::newTransform(_ctx, tType); transform->setSkipFlags(skipFlags); _buffer->_index = 0; // Inverse transform bool res = transform->inverse(*_buffer, *_data, preTransformLength); DELETE_AND_NULL(TransformSequence, transform); if (res == false) { storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); return T(*_data, blockId, 0, checksum1, Error::ERR_PROCESS_BLOCK, "Transform inverse failed"); } const int decoded = _data->_index - savedIdx; // Verify checksum if (_hasher32 != nullptr) { const uint32 checksum2 = _hasher32->hash(&_data->_array[savedIdx], decoded); if (checksum2 != uint32(checksum1)) { storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); stringstream ss; ss << "Corrupted bitstream: expected checksum " << std::hex << checksum1 << ", found " << std::hex << checksum2; return T(*_data, blockId, decoded, checksum1, Error::ERR_CRC_CHECK, ss.str()); } } else if (_hasher64 != nullptr) { const uint64 checksum2 = _hasher64->hash(&_data->_array[savedIdx], decoded); if (checksum2 != checksum1) { storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); stringstream ss; ss << "Corrupted bitstream: expected checksum " << std::hex << checksum1 << ", found " << std::hex << checksum2; return T(*_data, blockId, decoded, checksum1, Error::ERR_CRC_CHECK, ss.str()); } } return T(*_data, blockId, decoded, checksum1, 0, "Success"); } catch (const exception& e) { // Cancel any in-flight task waiting on this block. storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); if (transform != nullptr) { try { DELETE_AND_NULL(TransformSequence, transform); } catch (...) { } } if (ed != nullptr) { try { DELETE_AND_NULL(EntropyDecoder, ed); } catch (...) { } } if ((streamPerTask == true) && (ibs != nullptr)) { try { DELETE_AND_NULL(InputBitStream, ibs); } catch (...) { } } return T(*_data, blockId, 0, checksum1, Error::ERR_PROCESS_BLOCK, e.what()); } catch (...) { // Cancel any in-flight task waiting on this block. storeProcessedBlockId(CompressedInputStream::CANCEL_TASKS_ID); if (transform != nullptr) { try { DELETE_AND_NULL(TransformSequence, transform); } catch (...) { } } if (ed != nullptr) { try { DELETE_AND_NULL(EntropyDecoder, ed); } catch (...) { } } if ((streamPerTask == true) && (ibs != nullptr)) { try { DELETE_AND_NULL(InputBitStream, ibs); } catch (...) { } } return T(*_data, blockId, 0, checksum1, Error::ERR_PROCESS_BLOCK, "Unknown block processing error"); } } #undef DELETE_AND_NULL kanzi-cpp-2.6.0/src/io/CompressedInputStream.hpp000066400000000000000000000276031525576310500216430ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_CompressedInputStream #define knz_CompressedInputStream #include // definition of EOF #include #include #include "../concurrent.hpp" #include "../Context.hpp" #include "../Event.hpp" #include "../Listener.hpp" #include "../InputStream.hpp" #include "../SliceArray.hpp" #include "../bitstream/DefaultInputBitStream.hpp" #include "../util/XXHash.hpp" #if __cplusplus >= 201103L #include #endif #ifdef CONCURRENCY_ENABLED #include #endif namespace kanzi { class DecodingTaskResult FINAL { public: int _blockId; int _decoded; byte* _data; int _error; // 0 = OK std::string _msg; uint64 _checksum; bool _skipped; WallTimer::TimeData _completionTime; DecodingTaskResult() { _blockId = -1; _decoded = 0; _data = nullptr; _error = 0; _checksum = 0; _skipped = false; WallTimer timer; _completionTime = timer.getCurrentTime(); } DecodingTaskResult(const SliceArray& data, int blockId, int decoded, uint64 checksum, int error, const std::string& msg, bool skipped = false) : _blockId(blockId) , _decoded(decoded) , _data(data._array) , _error(error) , _msg(msg) , _checksum(checksum) , _skipped(skipped) { WallTimer timer; _completionTime = timer.getCurrentTime(); } DecodingTaskResult(const DecodingTaskResult& result) : _blockId(result._blockId) , _decoded(result._decoded) , _data(result._data) , _error(result._error) , _msg(result._msg) , _checksum(result._checksum) , _skipped(result._skipped) , _completionTime(result._completionTime) { } DecodingTaskResult& operator = (const DecodingTaskResult& result) { _msg = result._msg; _data = result._data; _blockId = result._blockId; _error = result._error; _decoded = result._decoded; _checksum = result._checksum; _completionTime = result._completionTime; _skipped = result._skipped; return *this; } #if __cplusplus >= 201103L // Check for C++11 or later DecodingTaskResult(DecodingTaskResult&& other) noexcept : _blockId(other._blockId) , _decoded(other._decoded) , _data(other._data) , _error(other._error) , _msg(std::move(other._msg)) , _checksum(other._checksum) , _skipped(other._skipped) , _completionTime(other._completionTime) { other._data = nullptr; } DecodingTaskResult& operator=(DecodingTaskResult&& other) noexcept { if (this != &other) { _blockId = other._blockId; _decoded = other._decoded; _data = other._data; // No ownership so don't need to delete _error = other._error; _msg = std::move(other._msg); _checksum = other._checksum; _skipped = other._skipped; _completionTime = other._completionTime; other._data = nullptr; } return *this; } #endif ~DecodingTaskResult() {} }; // A task used to decode a block // Several tasks (transform+entropy) may run in parallel template class DecodingTask FINAL : public Task { private: struct ParsedBlockHeader { byte _bytes[7]; uint _size; byte _skipFlags; int _preTransformLength; bool _transformedCopy; bool _rawCopy; ParsedBlockHeader() : _size(0) , _skipFlags(byte(0)) , _preTransformLength(0) , _transformedCopy(false) , _rawCopy(false) { } }; SliceArray* _data; SliceArray* _buffer; uint _blockLength; DefaultInputBitStream* _ibs; XXHash32* _hasher32; XXHash64* _hasher64; #ifdef CONCURRENCY_ENABLED std::mutex* _blockMutex; std::condition_variable* _blockCondition; #endif atomic_int_t* _processedBlockId; std::vector*> _listeners; Context _ctx; void storeProcessedBlockId(int value); int readBlockHeader(InputBitStream* ibs, int bsVersion, uint64 encodedBlockLength, uint64 transformType, ParsedBlockHeader& header, std::string& msg); public: DecodingTask(SliceArray* iBuffer, SliceArray* oBuffer, int blockSize, DefaultInputBitStream* ibs, XXHash32* hasher32, XXHash64* hasher64, #ifdef CONCURRENCY_ENABLED std::mutex* blockMutex, std::condition_variable* blockCondition, #endif atomic_int_t* processedBlockId, std::vector*>& listeners, const Context& ctx); ~DecodingTask(){} T run(); }; class CompressedInputStream : public InputStream { friend class DecodingTask; public: // If headerless == false, all provided compression parameters will be overwritten // with values read from the bitstream header. CompressedInputStream(InputStream& is, int jobs = 1, const std::string& entropy = "NONE", const std::string& transform = "NONE", int blockSize = 4*1024*1024, int checksum = 0, uint64 originalSize = 0, #ifdef CONCURRENCY_ENABLED ThreadPool* pool = nullptr, #endif bool headerless = false, int bsVersion = BITSTREAM_FORMAT_VERSION); // If headerless == true, the context must contain "entropy", "transform", "checksum" & "blockSize" // If "bsVersion" is missing, the current value of BITSTREAM_FORMAT_VERSION is assumed. CompressedInputStream(InputStream& is, Context& ctx, bool headerless = false); ~CompressedInputStream(); bool addListener(Listener& bl); bool removeListener(Listener& bl); std::streampos tellg(); std::istream& seekg(std::streampos pos); std::istream& putback(char c); std::istream& unget(); std::istream& read(char* s, std::streamsize n); std::streamsize gcount() const { return _gcount; } int get(); int peek(); void close(); uint64 getRead() const { return (_ibs->read() + 7) >> 3; } #if !defined(_MSC_VER) || _MSC_VER > 1500 bool seek(int64 bitPos); int64 tell(); #endif protected: void readHeader(); private: static const int BITSTREAM_TYPE; static const int BITSTREAM_FORMAT_VERSION; static const int DEFAULT_BUFFER_SIZE; static const int EXTRA_BUFFER_SIZE; static const byte COPY_BLOCK_MASK; static const byte TRANSFORMS_MASK; static const int MIN_BITSTREAM_BLOCK_SIZE; static const int MAX_BITSTREAM_BLOCK_SIZE; static const int TRANSFORMED_COPY_VERSION; static const int CANCEL_TASKS_ID; static const int MAX_CONCURRENCY; static const int MAX_BLOCK_ID; static inline uint32 mix32_v6(uint32 checksum, uint32 hash, uint32 value) { return checksum ^ (hash * uint32(~value)); } static inline uint32 mix32_v7(uint32 checksum, uint32 hash, uint32 value) { checksum ^= hash * uint32(~value); checksum = (checksum << 13) | (checksum >> 19); return checksum * 5 + 0x52DCE729; } int _blockSize; int _bufferId; // index of current read buffer int _maxBufferId; // max index of read buffer int _nbInputBlocks; int _jobs; int _bufferThreshold; int64 _available; // decoded not consumed bytes int64 _outputSize; XXHash32* _hasher32; XXHash64* _hasher64; SliceArray** _buffers; // input & output per block short _entropyType; uint64 _transformType; DefaultInputBitStream* _ibs; atomic_int_t _initialized; atomic_int_t _closed; atomic_int_t _blockId; atomic_int_t _submitBlockId; // Next block to submit to pool int _consumeBlockId; // Next block to be consumed by read() std::vector*> _listeners; std::streamsize _gcount; Context _ctx; Context* _parentCtx; // not owner bool _headless; std::vector _jobsPerTask; #ifdef CONCURRENCY_ENABLED ThreadPool* _pool; std::vector> _futures; std::mutex _blockMutex; std::condition_variable _blockCondition; #else std::vector _results; #endif void submitBlock(int bufferId); int _get(int inc); static void notifyListeners(std::vector*>& listeners, const Event& evt); }; inline int CompressedInputStream::get() { const int res = _get(1); _gcount = (res != EOF) ? 1 : 0; return res; } inline int CompressedInputStream::peek() { return _get(0); } inline std::streampos CompressedInputStream::tellg() { throw std::ios_base::failure("Not supported"); } inline std::istream& CompressedInputStream::seekg(std::streampos) { throw std::ios_base::failure("Not supported"); } inline std::istream& CompressedInputStream::putback(char) { setstate(std::ios::badbit); throw std::ios_base::failure("Not supported"); } inline std::istream& CompressedInputStream::unget() { setstate(std::ios::badbit); throw std::ios_base::failure("Not supported"); } #if !defined(_MSC_VER) || _MSC_VER > 1500 inline bool CompressedInputStream::seek(int64 bitPos) { // The only valid positions are block boundaries. if (LOAD_ATOMIC(_closed) == 1) return false; if (bitPos < 0) return false; #ifdef CONCURRENCY_ENABLED // Cancel any in-flight decode pipeline tied to the previous position. STORE_ATOMIC(_blockId, CANCEL_TASKS_ID); // Drain futures so no task can still consume the old underlying bitstream. for (int i = 0; i < _jobs; i++) { if (_futures[i].valid()) { try { (void) _futures[i].get(); } catch (...) { // Ignore: we are resetting the stream state anyway. } } } #endif // Reset decode state. _available = 0; _gcount = 0; _bufferId = 0; _maxBufferId = 0; _submitBlockId = 0; _consumeBlockId = 0; STORE_ATOMIC(_blockId, 0); if (_ibs->seek(bitPos) == false) return false; // Clear eof/fail flags potentially set by prior reads. this->clear(); // If stream was already initialized, bootstrap decoding tasks from new pos now. // If not initialized, read()/get() will initialize and submit as usual. if (LOAD_ATOMIC(_initialized) == 1) { for (int i = 0; i < _jobs; i++) submitBlock(i); } return true; } inline int64 CompressedInputStream::tell() { return _ibs->tell(); } #endif } #endif kanzi-cpp-2.6.0/src/io/CompressedOutputStream.cpp000066400000000000000000001141371525576310500220360ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include "CompressedOutputStream.hpp" #include "IOException.hpp" #include "../Error.hpp" #include "../Magic.hpp" #include "../entropy/EntropyEncoderFactory.hpp" #include "../entropy/EntropyUtils.hpp" #include "../transform/TransformFactory.hpp" #include "../util/fixedbuf.hpp" using namespace kanzi; using namespace std; // Clear ownership before delete so a throwing destructor cannot leave a // dangling non-null pointer for later cleanup. #define DELETE_AND_NULL(TYPE, PTR) \ do { \ TYPE* knzTmpPtr = (PTR); \ (PTR) = nullptr; \ delete knzTmpPtr; \ } while (0) const int CompressedOutputStream::BITSTREAM_TYPE = 0x4B414E5A; // "KANZ" const int CompressedOutputStream::BITSTREAM_FORMAT_VERSION = 7; const int CompressedOutputStream::DEFAULT_BUFFER_SIZE = 256 * 1024; const kanzi::byte CompressedOutputStream::COPY_BLOCK_MASK = kanzi::byte(0x80); const kanzi::byte CompressedOutputStream::TRANSFORMS_MASK = kanzi::byte(0x10); const int CompressedOutputStream::MIN_BITSTREAM_BLOCK_SIZE = 1024; const int CompressedOutputStream::MAX_BITSTREAM_BLOCK_SIZE = 1024 * 1024 * 1024; const int CompressedOutputStream::SMALL_BLOCK_SIZE = 15; const int CompressedOutputStream::CANCEL_TASKS_ID = -1; const int CompressedOutputStream::MAX_CONCURRENCY = 64; CompressedOutputStream::CompressedOutputStream(OutputStream& os, int tasks, const string& entropy, const string& transform, int blockSize, int checksum, uint64 fileSize, #ifdef CONCURRENCY_ENABLED ThreadPool* pool, #endif bool headerless) : OutputStream(os.rdbuf()) { #ifdef CONCURRENCY_ENABLED if ((tasks <= 0) || (tasks > MAX_CONCURRENCY)) { stringstream ss; ss << "The number of jobs must be in [1.." << MAX_CONCURRENCY << "], got " << tasks; throw invalid_argument(ss.str()); } _pool = pool; // can be null #else if (tasks != 1) throw invalid_argument("The number of jobs is limited to 1 in this version"); #endif if (blockSize > MAX_BITSTREAM_BLOCK_SIZE) { std::stringstream ss; ss << "The block size must be at most " << (MAX_BITSTREAM_BLOCK_SIZE >> 20) << " MB"; throw invalid_argument(ss.str()); } if (blockSize < MIN_BITSTREAM_BLOCK_SIZE) { std::stringstream ss; ss << "The block size must be at least " << MIN_BITSTREAM_BLOCK_SIZE; throw invalid_argument(ss.str()); } if ((blockSize & -16) != blockSize) throw invalid_argument("The block size must be a multiple of 16"); _blockId = 0; _inputBlockId = 0; _bufferId = 0; _blockSize = blockSize; _bufferThreshold = blockSize; _inputSize = fileSize; const int nbBlocks = (_inputSize == 0) ? 0 : int((_inputSize + int64(blockSize - 1)) / int64(blockSize)); _nbInputBlocks = min(nbBlocks, MAX_CONCURRENCY); _headless = headerless; _initialized = 0; _closed = 0; _obs = new DefaultOutputBitStream(os, DEFAULT_BUFFER_SIZE); _entropyType = EntropyEncoderFactory::getType(entropy.c_str()); _transformType = TransformFactory::getType(transform.c_str()); if (checksum == 0) { _hasher32 = nullptr; _hasher64 = nullptr; } else if (checksum == 32) { _hasher32 = new XXHash32(BITSTREAM_TYPE); _hasher64 = nullptr; } else if (checksum == 64) { _hasher32 = nullptr; _hasher64 = new XXHash64(BITSTREAM_TYPE); } else { throw invalid_argument("The block checksum size must be 0, 32 or 64"); } _jobs = tasks; _ctx.putInt("blockSize", _blockSize); _ctx.putInt("checksum", checksum); _ctx.putString("entropy", entropy); _ctx.putString("transform", transform); _ctx.putInt("bsVersion", BITSTREAM_FORMAT_VERSION); #ifdef CONCURRENCY_ENABLED _futures.resize(_jobs); #endif _jobsPerTask.resize(_jobs); // Assign optimal number of tasks and jobs per task (if the number of blocks is available) if (_jobs > 1) { // Limit the number of tasks if there are fewer blocks that _jobs // It allows more jobs per task and reduces memory usage. int nbTasks = (_nbInputBlocks != 0) ? min(_nbInputBlocks, _jobs) : _jobs; Global::computeJobsPerTask(&_jobsPerTask[0], _jobs, nbTasks); } else { _jobsPerTask[0] = 1; } // Allocate first buffer and add padding for incompressible blocks _buffers = new SliceArray*[2 * _jobs]; const int bufSize = max(_blockSize + (_blockSize >> 3), DEFAULT_BUFFER_SIZE); _buffers[0] = new SliceArray(new kanzi::byte[bufSize], bufSize, 0); for (int i = 1; i < 2 * _jobs; i++) _buffers[i] = new SliceArray(nullptr, 0, 0); } CompressedOutputStream::CompressedOutputStream(OutputStream& os, Context& ctx, bool headerless) : OutputStream(os.rdbuf()) , _ctx(ctx) { int tasks = ctx.getInt("jobs", 1); #ifdef CONCURRENCY_ENABLED if ((tasks <= 0) || (tasks > MAX_CONCURRENCY)) { stringstream ss; ss << "The number of jobs must be in [1.." << MAX_CONCURRENCY << "], got " << tasks; throw invalid_argument(ss.str()); } _pool = _ctx.getPool(); // can be null #else if (tasks != 1) throw invalid_argument("The number of jobs is limited to 1 in this version"); #endif int blockSize = ctx.getInt("blockSize"); if (blockSize > MAX_BITSTREAM_BLOCK_SIZE) { std::stringstream ss; ss << "The block size must be at most " << (MAX_BITSTREAM_BLOCK_SIZE >> 20) << " MB"; throw invalid_argument(ss.str()); } if (blockSize < MIN_BITSTREAM_BLOCK_SIZE) { std::stringstream ss; ss << "The block size must be at least " << MIN_BITSTREAM_BLOCK_SIZE; throw invalid_argument(ss.str()); } if ((blockSize & -16) != blockSize) throw invalid_argument("The block size must be a multiple of 16"); _inputSize = ctx.getLong("fileSize", 0); const int nbBlocks = (_inputSize == 0) ? 0 : int((_inputSize + int64(blockSize - 1)) / int64(blockSize)); _nbInputBlocks = min(nbBlocks, MAX_CONCURRENCY); _jobs = tasks; _blockId = 0; _inputBlockId = 0; _bufferId = 0; _blockSize = blockSize; _bufferThreshold = blockSize; _initialized = 0; _closed = 0; _headless = headerless; _obs = new DefaultOutputBitStream(os, DEFAULT_BUFFER_SIZE); _ctx.putInt("bsVersion", BITSTREAM_FORMAT_VERSION); string entropyCodec = ctx.getString("entropy"); string transform = ctx.getString("transform"); _entropyType = EntropyEncoderFactory::getType(entropyCodec.c_str()); _transformType = TransformFactory::getType(transform.c_str()); int checksum = ctx.getInt("checksum", 0); if (checksum == 0) { _hasher32 = nullptr; _hasher64 = nullptr; } else if (checksum == 32) { _hasher32 = new XXHash32(BITSTREAM_TYPE); _hasher64 = nullptr; } else if (checksum == 64) { _hasher32 = nullptr; _hasher64 = new XXHash64(BITSTREAM_TYPE); } else { throw invalid_argument("The block checksum size must be 0, 32 or 64"); } #ifdef CONCURRENCY_ENABLED _futures.resize(_jobs); #endif _jobsPerTask.resize(_jobs); // Assign optimal number of tasks and jobs per task (if the number of blocks is available) if (_jobs > 1) { // Limit the number of tasks if there are fewer blocks that _jobs // It allows more jobs per task and reduces memory usage. int nbTasks = (_nbInputBlocks != 0) ? min(_nbInputBlocks, _jobs) : _jobs; Global::computeJobsPerTask(&_jobsPerTask[0], _jobs, nbTasks); } else { _jobsPerTask[0] = 1; } _buffers = new SliceArray*[2 * _jobs]; // Allocate first buffer and add padding for incompressible blocks const int bufSize = max(_blockSize + (_blockSize >> 3), DEFAULT_BUFFER_SIZE); _buffers[0] = new SliceArray(new kanzi::byte[bufSize], bufSize, 0); for (int i = 1; i < 2 * _jobs; i++) _buffers[i] = new SliceArray(nullptr, 0, 0); } CompressedOutputStream::~CompressedOutputStream() { try { close(); } catch (const exception&) { // Ignore and continue } for (int i = 0; i < 2 * _jobs; i++) { if (_buffers[i]->_array != nullptr) delete[] _buffers[i]->_array; delete _buffers[i]; } delete[] _buffers; delete _obs; if (_hasher32 != nullptr) { delete _hasher32; _hasher32 = nullptr; } if (_hasher64 != nullptr) { delete _hasher64; _hasher64 = nullptr; } } void CompressedOutputStream::writeHeader() { if ((_headless == true) || (EXCHANGE_ATOMIC(_initialized, 1) == 1)) return; if (_obs->writeBits(BITSTREAM_TYPE, 32) != 32) throw IOException("Cannot write bitstream type to header", Error::ERR_WRITE_FILE); if (_obs->writeBits(BITSTREAM_FORMAT_VERSION, 4) != 4) throw IOException("Cannot write bitstream version to header", Error::ERR_WRITE_FILE); uint ckSize = 0; if (_hasher32 != nullptr) ckSize = 1; else if (_hasher64 != nullptr) ckSize = 2; if (_obs->writeBits(ckSize, 2) != 2) throw IOException("Cannot write block checksum size to header", Error::ERR_WRITE_FILE); if (_obs->writeBits(_entropyType, 5) != 5) throw IOException("Cannot write entropy type to header", Error::ERR_WRITE_FILE); if (_obs->writeBits(_transformType, 48) != 48) throw IOException("Cannot write transform types to header", Error::ERR_WRITE_FILE); if (_obs->writeBits(_blockSize >> 4, 28) != 28) throw IOException("Cannot write block size to header", Error::ERR_WRITE_FILE); // _inputSize not provided or >= 2^48 -> 0, <2^16 -> 1, <2^32 -> 2, <2^48 -> 3 const uint szMask = ((_inputSize == 0) || (_inputSize >= (int64(1) << 48))) ? 0 : (Global::log2(uint64(_inputSize)) >> 4) + 1; if (_obs->writeBits(szMask, 2) != 2) throw IOException("Cannot write size of input to header", Error::ERR_WRITE_FILE); if (szMask != 0) { if (_obs->writeBits(_inputSize, 16 * szMask) != 16 * szMask) throw IOException("Cannot write size of input to header", Error::ERR_WRITE_FILE); } const uint64 padding = 0; if (_obs->writeBits(padding, 15) != 15) throw IOException("Cannot write padding to header", Error::ERR_WRITE_FILE); uint32 seed = 0x01030507 * BITSTREAM_FORMAT_VERSION; // no const to avoid VS2008 warning const uint32 HASH = 0x1E35A7BD; uint32 cksum = HASH * seed; cksum = CompressedOutputStream::mix32(cksum, HASH, ckSize); cksum = CompressedOutputStream::mix32(cksum, HASH, uint32(_entropyType)); cksum = CompressedOutputStream::mix32(cksum, HASH, uint32(_transformType >> 32)); cksum = CompressedOutputStream::mix32(cksum, HASH, uint32(_transformType)); cksum = CompressedOutputStream::mix32(cksum, HASH, uint32(_blockSize)); if (szMask != 0) { cksum = CompressedOutputStream::mix32(cksum, HASH, uint32(_inputSize >> 32)); cksum = CompressedOutputStream::mix32(cksum, HASH, uint32(_inputSize)); } cksum = (cksum >> 23) ^ (cksum >> 3); if (_obs->writeBits(uint64(cksum & 0xFFFFFFu), 24) != 24) throw IOException("Cannot write checksum to header", Error::ERR_WRITE_FILE); } bool CompressedOutputStream::addListener(Listener& bl) { _listeners.push_back(&bl); return true; } bool CompressedOutputStream::removeListener(Listener& bl) { std::vector*>::iterator it = find(_listeners.begin(), _listeners.end(), &bl); if (it == _listeners.end()) return false; _listeners.erase(it); return true; } ostream& CompressedOutputStream::write(const char* data, streamsize length) { if (length < 0) throw IOException("Invalid buffer size"); streamsize off = 0; streamsize remaining = length; while (remaining > 0) { const streamsize lenChunk = min(remaining, streamsize(_bufferThreshold - _buffers[_bufferId]->_index)); if (lenChunk > 0) { memcpy(&_buffers[_bufferId]->_array[_buffers[_bufferId]->_index], &data[off], size_t(lenChunk)); _buffers[_bufferId]->_index += int(lenChunk); off += lenChunk; remaining -= lenChunk; if (_buffers[_bufferId]->_index >= _bufferThreshold) { processBuffer(); } } else { // Handle full buffer / closed stream processBuffer(); } } return *this; } void CompressedOutputStream::close() { if (LOAD_ATOMIC(_closed) == 1) return; string errMsg; int errCode = Error::ERR_WRITE_FILE; try { // Submit the last partial block (if any) submitBlock(); #ifdef CONCURRENCY_ENABLED // Wait for ALL pending tasks to complete for (int i = 0; i < _jobs; i++) { if (_futures[i].valid()) { try { EncodingTaskResult res = _futures[i].get(); // Keep the first real worker error. Later tasks may only // report cancellation or a secondary cleanup failure. if ((res._error != 0) && (errMsg == "")) { errMsg = res._msg; errCode = res._error; } } catch (const exception& e) { if (errMsg == "") { errMsg = e.what(); errCode = Error::ERR_PROCESS_BLOCK; } } catch (...) { if (errMsg == "") { errMsg = "Unknown asynchronous block processing error"; errCode = Error::ERR_PROCESS_BLOCK; } } } } #endif if (errMsg == "") { // Write last block: length-3 (0) and 0 bits _obs->writeBits(uint64(0), 5); _obs->writeBits(uint64(0), 3); _obs->close(); } } catch (const IOException& e) { setstate(ios::badbit); errMsg = e.what(); errCode = e.error(); } catch (const exception& e) { setstate(ios::badbit); errMsg = e.what(); } catch (...) { setstate(ios::badbit); errMsg = "Unknown compressed stream error"; errCode = Error::ERR_WRITE_FILE; } if (errMsg != "") setstate(ios::badbit); STORE_ATOMIC(_closed, 1); // Force subsequent writes to trigger submitBlock immediately _bufferThreshold = 0; // Release resources for (int i = 0; i < 2 * _jobs; i++) { if (_buffers[i]->_array != nullptr) delete[] _buffers[i]->_array; _buffers[i]->_array = nullptr; _buffers[i]->_length = 0; _buffers[i]->_index = 0; } if (errMsg != "") throw IOException(errMsg, errCode); setstate(ios::eofbit); } void CompressedOutputStream::processBuffer() { submitBlock(); _bufferId = (_bufferId + 1) % _jobs; #ifdef CONCURRENCY_ENABLED if (_futures[_bufferId].valid()) { try { EncodingTaskResult res = _futures[_bufferId].get(); if (res._error != 0) { throw IOException(res._msg, res._error); } } catch (...) { // Do not let another worker access buffers or the shared bitstream // after the caller starts destroying this stream. drainTasks(); throw; } } #endif const int bSize = _blockSize + (_blockSize >> 6); const int bufSize = max(bSize, 65536); if (_buffers[_bufferId]->_length == 0) { if (_buffers[_bufferId]->_array != nullptr) delete[] _buffers[_bufferId]->_array; _buffers[_bufferId]->_array = new kanzi::byte[bufSize]; _buffers[_bufferId]->_length = bufSize; } _buffers[_bufferId]->_index = 0; } #ifdef CONCURRENCY_ENABLED void CompressedOutputStream::drainTasks() noexcept { for (int i = 0; i < _jobs; i++) { if (_futures[i].valid()) { try { _futures[i].get(); } catch (...) { // The caller already has the primary error. The purpose of // this method is to complete every worker before teardown. } } } } #endif void CompressedOutputStream::submitBlock() { if (LOAD_ATOMIC(_closed) == 1) throw IOException("Stream closed", Error::ERR_WRITE_FILE); writeHeader(); // Ensure header is written before first block processing const int dataLength = _buffers[_bufferId]->_index; if (dataLength == 0) return; // Increment input block counter (1-based for the Task logic) _inputBlockId++; Context copyCtx(_ctx); copyCtx.putLong("tType", _transformType); copyCtx.putInt("eType", _entropyType); copyCtx.putInt("blockId", _inputBlockId); copyCtx.putInt("size", dataLength); copyCtx.putInt("jobs", _jobsPerTask[_bufferId]); // Prepare the buffer for processing _buffers[_bufferId]->_index = 0; // Create the task // Note: Input is _buffers[_bufferId], Output is _buffers[_jobs + _bufferId] EncodingTask* task = new EncodingTask( _buffers[_bufferId], _buffers[_jobs + _bufferId], _obs, _hasher32, _hasher64, #ifdef CONCURRENCY_ENABLED &_blockMutex, &_blockCondition, #endif &_blockId, _listeners, copyCtx); #ifdef CONCURRENCY_ENABLED std::shared_ptr> safeTask(task); auto taskWrapper = [safeTask]() { return safeTask->run(); }; if (_pool == nullptr) { _futures[_bufferId] = std::async(std::launch::async, taskWrapper); } else { // REQUIRES: Pool size > Number of concurrent tasks to avoid deadlock _futures[_bufferId] = _pool->schedule(taskWrapper); } #else // Synchronous fallback try { EncodingTaskResult res = task->run(); if (res._error != 0) throw IOException(res._msg, res._error); } catch (...) { delete task; throw; } delete task; #endif } ostream& CompressedOutputStream::put(char c) { try { if (_buffers[_bufferId]->_index >= _bufferThreshold) { // Submit current buffer submitBlock(); // Rotate to next buffer _bufferId = (_bufferId + 1) % _jobs; // If concurrent, wait if the target buffer is still busy #ifdef CONCURRENCY_ENABLED if (_futures[_bufferId].valid()) { EncodingTaskResult res = _futures[_bufferId].get(); if (res._error != 0) throw IOException(res._msg, res._error); } #endif // Allocation / Reset logic const int bufSize = max(_blockSize + (_blockSize >> 6), 65536); if (_buffers[_bufferId]->_length == 0) { if (_buffers[_bufferId]->_array != nullptr) delete[] _buffers[_bufferId]->_array; _buffers[_bufferId]->_array = new kanzi::byte[bufSize]; _buffers[_bufferId]->_length = bufSize; } _buffers[_bufferId]->_index = 0; } _buffers[_bufferId]->_array[_buffers[_bufferId]->_index++] = kanzi::byte(c); return *this; } catch (const exception& e) { setstate(std::ios::badbit); throw std::ios_base::failure(e.what()); } } void CompressedOutputStream::notifyListeners(vector*>& listeners, const Event& evt) { for (vector*>::iterator it = listeners.begin(); it != listeners.end(); ++it) (*it)->processEvent(evt); } template EncodingTask::EncodingTask(SliceArray* iBuffer, SliceArray* oBuffer, DefaultOutputBitStream* obs, XXHash32* hasher32, XXHash64* hasher64, #ifdef CONCURRENCY_ENABLED std::mutex* blockMutex, std::condition_variable* blockCondition, #endif atomic_int_t* processedBlockId, vector*>& listeners, const Context& ctx) : _obs(obs) , _listeners(listeners) , _ctx(ctx) { _data = iBuffer; _buffer = oBuffer; _hasher32 = hasher32; _hasher64 = hasher64; #ifdef CONCURRENCY_ENABLED _blockMutex = blockMutex; _blockCondition = blockCondition; #endif _processedBlockId = processedBlockId; } template void EncodingTask::storeProcessedBlockId(int value) { #ifdef CONCURRENCY_ENABLED { std::lock_guard lock(*_blockMutex); STORE_ATOMIC(*_processedBlockId, value); } _blockCondition->notify_all(); #else STORE_ATOMIC(*_processedBlockId, value); #endif } template void EncodingTask::fetchAddProcessedBlockId() { #ifdef CONCURRENCY_ENABLED { std::lock_guard lock(*_blockMutex); FETCH_ADD_ATOMIC(*_processedBlockId, 1); } _blockCondition->notify_all(); #else FETCH_ADD_ATOMIC(*_processedBlockId, 1); #endif } // Encode mode + transformed entropy coded data // mode | 0b1yy0xxxx => raw copy block // | 0b1yy1xxxx => transformed copy block (version >= 7) // | 0b0yy00000 => size(size(block))-1 // case 4 transforms or less // | 0b0001xxxx => transform sequence skip flags (1 means skip) // case more than 4 transforms // | 0b0yy00000 0bxxxxxxxx => transform sequence skip flags in next kanzi::byte (1 means skip) template T EncodingTask::run() { const int blockId = _ctx.getInt("blockId"); const int blockLength = _ctx.getInt("size"); TransformSequence* transform = nullptr; EntropyEncoder* ee = nullptr; try { if (blockLength == 0) { // Last block (only block with 0 length) fetchAddProcessedBlockId(); return T(blockId, 0, "Success"); } kanzi::byte mode = kanzi::byte(0); int postTransformLength = blockLength; uint64 checksum = 0; uint64 tType = _ctx.getLong("tType"); short eType = short(_ctx.getInt("eType")); Event::HashType hashType = Event::NO_HASH; WallTimer timer; // Compute block checksum if (_hasher32 != nullptr) { checksum = _hasher32->hash(&_data->_array[_data->_index], blockLength); hashType = Event::SIZE_32; } else if (_hasher64 != nullptr) { checksum = _hasher64->hash(&_data->_array[_data->_index], blockLength); hashType = Event::SIZE_64; } if (_listeners.size() > 0) { // Notify before transform Event evt(Event::BEFORE_TRANSFORM, blockId, int64(blockLength), timer.getCurrentTime(), checksum, hashType); CompressedOutputStream::notifyListeners(_listeners, evt); } if (blockLength <= CompressedOutputStream::SMALL_BLOCK_SIZE) { tType = TransformFactory::NONE_TYPE; eType = EntropyEncoderFactory::NONE_TYPE; mode |= CompressedOutputStream::COPY_BLOCK_MASK; } else { int checkSkip = _ctx.getInt("skipBlocks", 0); if (checkSkip != 0) { bool skip = Magic::isCompressed(Magic::getType(&_data->_array[_data->_index])); if (skip == false) { uint histo[256] = { 0 }; Global::computeHistogram(&_data->_array[_data->_index], blockLength, histo); const int entropy = Global::computeOrder0Entropy1024(blockLength, histo); skip = entropy >= EntropyUtils::INCOMPRESSIBLE_THRESHOLD; //_ctx.putString("histo0", toString(histo, 256)); } if (skip == true) { tType = TransformFactory::NONE_TYPE; eType = EntropyEncoderFactory::NONE_TYPE; mode |= CompressedOutputStream::COPY_BLOCK_MASK; } } } _ctx.putInt("size", blockLength); transform = TransformFactory::newTransform(_ctx, tType); const int requiredSize = transform->getMaxEncodedLength(blockLength); if (blockLength >= 4) { uint magic = Magic::getType(&_data->_array[_data->_index]); if (Magic::isCompressed(magic) == true) _ctx.putInt("dataType", Global::BIN); else if (Magic::isMultimedia(magic) == true) _ctx.putInt("dataType", Global::MULTIMEDIA); else if (Magic::isExecutable(magic) == true) _ctx.putInt("dataType", Global::EXE); } if (_buffer->_length < requiredSize) { if (_buffer->_array != nullptr) delete[] _buffer->_array; _buffer->_array = new kanzi::byte[requiredSize]; _buffer->_length = requiredSize; } // Forward transform (ignore error, encode skipFlags) // _data->_length is at least blockLength _buffer->_index = 0; transform->forward(*_data, *_buffer, blockLength); const int nbTransforms = transform->getNbTransforms(); const kanzi::byte skipFlags = transform->getSkipFlags(); DELETE_AND_NULL(TransformSequence, transform); postTransformLength = _buffer->_index; if (postTransformLength < 0) { storeProcessedBlockId(CompressedOutputStream::CANCEL_TASKS_ID); return T(blockId, Error::ERR_WRITE_FILE, "Invalid transform size"); } _ctx.putInt("size", postTransformLength); const int dataSize = (postTransformLength < 256) ? 1 : (Global::_log2(uint32(postTransformLength)) >> 3) + 1; if (dataSize > 4) { storeProcessedBlockId(CompressedOutputStream::CANCEL_TASKS_ID); return T(blockId, Error::ERR_WRITE_FILE, "Invalid block data length"); } // Record size of 'block size' - 1 in bytes mode |= kanzi::byte(((dataSize - 1) & 0x03) << 5); // Binary entropy codecs reject blocks at or above 1 GiB. The // transformed-copy representation can preserve the transformed data // without passing it through entropy coding. bool transformedCopy = false; if ((mode & CompressedOutputStream::COPY_BLOCK_MASK) == kanzi::byte(0)) { if (uint(postTransformLength) >= uint(CompressedOutputStream::MAX_BITSTREAM_BLOCK_SIZE)) { transformedCopy = true; mode |= CompressedOutputStream::COPY_BLOCK_MASK | CompressedOutputStream::TRANSFORMS_MASK; } } if (_listeners.size() > 0) { // Notify after transform Event evt(Event::AFTER_TRANSFORM, blockId, int64(postTransformLength), timer.getCurrentTime(), checksum, hashType); CompressedOutputStream::notifyListeners(_listeners, evt); } const int bufSize = max(CompressedOutputStream::DEFAULT_BUFFER_SIZE, max(postTransformLength, blockLength + (blockLength >> 3))); if (_data->_length < bufSize) { // Rare case where the transform expanded the input or // entropy coder may expand size. delete[] _data->_array; _data->_length = bufSize; _data->_array = new kanzi::byte[_data->_length]; } _data->_index = 0; growable_ofixedbuf buf(_data); ostream os(&buf); DefaultOutputBitStream obs(os); // Write block 'header' (mode + compressed length) kanzi::byte headerSkipFlags = skipFlags; if (transformedCopy == true) { if (nbTransforms <= 4) { mode |= kanzi::byte(skipFlags >> 4); headerSkipFlags = (mode << 4) | kanzi::byte(0x0F); } else { headerSkipFlags = skipFlags; } obs.writeBits(uint64(mode), 8); if (nbTransforms > 4) obs.writeBits(uint64(skipFlags), 8); } else if (((mode & CompressedOutputStream::COPY_BLOCK_MASK) != kanzi::byte(0)) || (nbTransforms <= 4)) { mode |= kanzi::byte(skipFlags >> 4); if ((mode & CompressedOutputStream::COPY_BLOCK_MASK) != kanzi::byte(0)) headerSkipFlags = kanzi::byte(0); else headerSkipFlags = (mode << 4) | kanzi::byte(0x0F); obs.writeBits(uint64(mode), 8); } else { mode |= CompressedOutputStream::TRANSFORMS_MASK; obs.writeBits(uint64(mode), 8); obs.writeBits(uint64(skipFlags), 8); } obs.writeBits(postTransformLength, 8 * dataSize); // Reserve the block header checksum byte. The encoded block length is // only known after entropy coding, so patch this byte once the complete // temporary block has been written. uint headerChecksumIndex = uint(1 + dataSize); if (((mode & CompressedOutputStream::TRANSFORMS_MASK) != kanzi::byte(0)) && (nbTransforms > 4)) headerChecksumIndex++; obs.writeBits(uint64(0), 8); // Write checksum if (_hasher32 != nullptr) obs.writeBits(checksum, 32); else if (_hasher64 != nullptr) obs.writeBits(checksum, 64); if (_listeners.size() > 0) { // Notify before entropy Event evt(Event::BEFORE_ENTROPY, blockId, int64(postTransformLength), timer.getCurrentTime(), checksum, hashType); CompressedOutputStream::notifyListeners(_listeners, evt); } uint64 written = 0; if (transformedCopy == true) { // Entropy coding is unavailable for this transformed block. Keep // the transformed bytes and let the decoder reverse the transforms. for (uint n = 0, remaining = uint(postTransformLength); remaining > 0; ) { const uint chunk = min(remaining, uint(1) << 27); obs.writeBits(&_buffer->_array[n], 8 * chunk); n += chunk; remaining -= chunk; } obs.close(); written = obs.written(); } else { // Each block is encoded separately // Rebuild the entropy encoder to reset block statistics ee = EntropyEncoderFactory::newEncoder(obs, _ctx, eType); // Entropy encode block if (ee->encode(_buffer->_array, 0, postTransformLength) != postTransformLength) { try { DELETE_AND_NULL(EntropyEncoder, ee); } catch (...) { } storeProcessedBlockId(CompressedOutputStream::CANCEL_TASKS_ID); return T(blockId, Error::ERR_PROCESS_BLOCK, "Entropy coding failed"); } // Dispose before processing statistics (may write to the bitstream) ee->dispose(); DELETE_AND_NULL(EntropyEncoder, ee); obs.close(); written = obs.written(); } // If not a copy block, check if entropy expanded the transformed block if ((mode & CompressedOutputStream::COPY_BLOCK_MASK) == kanzi::byte(0)) { const uint64 rawPayloadBytes = uint64(postTransformLength); const uint64 entropyPayloadBytes = (written + 7) >> 3; // If entropy coding expanded the block, rebuild the temporary payload as // a "transformed copy" block. This keeps the transform gains while // bypassing the entropy coder for pathological cases. if (rawPayloadBytes < entropyPayloadBytes) { _data->_index = 0; growable_ofixedbuf copyBuf(_data); ostream copyOs(©Buf); DefaultOutputBitStream copyObs(copyOs); kanzi::byte copyMode = kanzi::byte(mode | CompressedOutputStream::COPY_BLOCK_MASK | CompressedOutputStream::TRANSFORMS_MASK); copyObs.writeBits(uint64(copyMode), 8); if (nbTransforms > 4) copyObs.writeBits(uint64(skipFlags), 8); copyObs.writeBits(postTransformLength, 8 * dataSize); headerChecksumIndex = uint(1 + dataSize); if (nbTransforms > 4) { headerChecksumIndex++; headerSkipFlags = skipFlags; } else { headerSkipFlags = (copyMode << 4) | kanzi::byte(0x0F); } copyObs.writeBits(uint64(0), 8); if (_hasher32 != nullptr) copyObs.writeBits(checksum, 32); else if (_hasher64 != nullptr) copyObs.writeBits(checksum, 64); // Reuse the post-transform bytes already present in _buffer: // no entropy coding, just emit the transformed payload verbatim. for (uint n = 0, remaining = uint(postTransformLength); remaining > 0; ) { const uint chunk = min(remaining, uint(1) << 27); copyObs.writeBits(&_buffer->_array[n], 8 * chunk); n += chunk; remaining -= chunk; } copyObs.close(); written = copyObs.written(); mode = copyMode; } } // Protect the block header and its outer encoded bit length independently // from the optional checksum of the decoded payload. const uint32 hash = 0x1E35A7BDu; uint32 cksum = hash * 0x01030507u; cksum = CompressedOutputStream::mix32(cksum, hash, uint32(mode)); cksum = CompressedOutputStream::mix32(cksum, hash, uint32(uint8_t(headerSkipFlags))); cksum = CompressedOutputStream::mix32(cksum, hash, uint32(postTransformLength)); cksum = CompressedOutputStream::mix32(cksum, hash, uint32(written >> 32)); cksum = CompressedOutputStream::mix32(cksum, hash, uint32(written)); cksum = (cksum >> 23) ^ (cksum >> 3); _data->_array[headerChecksumIndex] = kanzi::byte(cksum & 0xFFu); const uint lw = (written < 8) ? 3 : uint(Global::log2(uint32(written >> 3)) + 4); #ifdef CONCURRENCY_ENABLED { std::unique_lock lock(*_blockMutex); _blockCondition->wait(lock, [this, blockId]() { const int taskId = LOAD_ATOMIC(*_processedBlockId); return (taskId == CompressedOutputStream::CANCEL_TASKS_ID) || (taskId == blockId - 1); }); } if (LOAD_ATOMIC(*_processedBlockId) == CompressedOutputStream::CANCEL_TASKS_ID) return T(blockId, 0, "Canceled"); #endif // Emit block size in bits (max size pre-entropy is 1 GB = 1 << 30 bytes) #if !defined(_MSC_VER) || _MSC_VER > 1500 const int64 blockOffset = _obs->tell(); #endif _obs->writeBits(lw - 3, 5); // write length-3 (5 bits max) _obs->writeBits(written, lw); int64 ww = int64((written + 7) >> 3); // Emit data to shared bitstream uint64 remaining = written; for (uint n = 0; remaining > 0; ) { uint chkSize = uint(min(remaining, uint64(1) << 30)); _obs->writeBits(&_data->_array[n], chkSize); n += ((chkSize + 7) >> 3); remaining -= uint64(chkSize); } // After completion of the entropy coding, increment the block id. // It unblocks the task processing the next block (if any). storeProcessedBlockId(blockId); if (_listeners.size() > 0) { // Notify after entropy Event evt1(Event::AFTER_ENTROPY, blockId, ww, timer.getCurrentTime(), checksum, hashType); CompressedOutputStream::notifyListeners(_listeners, evt1); #if !defined(_MSC_VER) || _MSC_VER > 1500 if (_ctx.getInt("verbosity", 0) > 4) { Event evt2(Event::BLOCK_INFO, blockId, int64((written + 7) >> 3), timer.getCurrentTime(), checksum, hashType, blockOffset, uint8(skipFlags)); CompressedOutputStream::notifyListeners(_listeners, evt2); } #endif } return T(blockId, 0, "Success"); } catch (const exception& e) { // Cancel any in-flight task waiting on this block. storeProcessedBlockId(CompressedOutputStream::CANCEL_TASKS_ID); // Cleanup must not replace the original block-processing error. In // particular, an encoder destructor may flush its bitstream and can // itself report a stream error while unwinding. if (transform != nullptr) { try { DELETE_AND_NULL(TransformSequence, transform); } catch (...) { } } if (ee != nullptr) { try { DELETE_AND_NULL(EntropyEncoder, ee); } catch (...) { } } return T(blockId, Error::ERR_PROCESS_BLOCK, e.what()); } catch (...) { // Keep the cancellation and cleanup guarantees even for an exception // type that does not derive from std::exception. storeProcessedBlockId(CompressedOutputStream::CANCEL_TASKS_ID); if (transform != nullptr) { try { DELETE_AND_NULL(TransformSequence, transform); } catch (...) { } } if (ee != nullptr) { try { DELETE_AND_NULL(EntropyEncoder, ee); } catch (...) { } } return T(blockId, Error::ERR_PROCESS_BLOCK, "Unknown block processing error"); } } #undef DELETE_AND_NULL kanzi-cpp-2.6.0/src/io/CompressedOutputStream.hpp000066400000000000000000000156631525576310500220470ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_CompressedOutputStream #define knz_CompressedOutputStream #include #include #include "../concurrent.hpp" #include "../Context.hpp" #include "../Event.hpp" #include "../Listener.hpp" #include "../OutputStream.hpp" #include "../SliceArray.hpp" #include "../bitstream/DefaultOutputBitStream.hpp" #include "../util/XXHash.hpp" #if __cplusplus >= 201103L #include #endif #ifdef CONCURRENCY_ENABLED #include #endif namespace kanzi { class EncodingTaskResult FINAL { public: int _blockId; int _error; // 0 = OK std::string _msg; EncodingTaskResult() { _blockId = -1; _error = 0; } EncodingTaskResult(int blockId, int error, const std::string& msg) : _blockId(blockId) , _error(error) , _msg(msg) { } EncodingTaskResult(const EncodingTaskResult& result) : _blockId(result._blockId) , _error(result._error) , _msg(result._msg) { } EncodingTaskResult& operator = (const EncodingTaskResult& result) { _msg = result._msg; _blockId = result._blockId; _error = result._error; return *this; } #if __cplusplus >= 201103L EncodingTaskResult(EncodingTaskResult&& other) noexcept : _blockId(other._blockId) , _error(other._error) , _msg(std::move(other._msg)) // Transfer ownership of string buffer { } // Move Assignment Operator EncodingTaskResult& operator=(EncodingTaskResult&& other) noexcept { if (this != &other) { _blockId = other._blockId; _error = other._error; _msg = std::move(other._msg); // Transfer ownership of string buffer } return *this; } #endif ~EncodingTaskResult() {} }; // A task used to encode a block // Several tasks (transform+entropy) may run in parallel template class EncodingTask FINAL : public Task { private: SliceArray* _data; SliceArray* _buffer; DefaultOutputBitStream* _obs; XXHash32* _hasher32; XXHash64* _hasher64; #ifdef CONCURRENCY_ENABLED std::mutex* _blockMutex; std::condition_variable* _blockCondition; #endif atomic_int_t* _processedBlockId; std::vector*> _listeners; Context _ctx; void storeProcessedBlockId(int value); void fetchAddProcessedBlockId(); public: EncodingTask(SliceArray* iBuffer, SliceArray* oBuffer, DefaultOutputBitStream* obs, XXHash32* hasher32, XXHash64* hasher64, #ifdef CONCURRENCY_ENABLED std::mutex* blockMutex, std::condition_variable* blockCondition, #endif atomic_int_t* processedBlockId, std::vector*>& listeners, const Context& ctx); ~EncodingTask(){} T run(); }; class CompressedOutputStream : public OutputStream { friend class EncodingTask; public: CompressedOutputStream(OutputStream& os, int jobs = 1, const std::string& entropy = "NONE", const std::string& transform = "NONE", int blockSize = 4*1024*1024, int checksum = 0, uint64 originalSize = 0, #ifdef CONCURRENCY_ENABLED ThreadPool* pool = nullptr, #endif bool headerless = false); CompressedOutputStream(OutputStream& os, Context& ctx, bool headerless = false); ~CompressedOutputStream(); bool addListener(Listener& bl); bool removeListener(Listener& bl); std::ostream& write(const char* s, std::streamsize n); std::ostream& put(char c); std::ostream& flush(); std::streampos tellp(); std::ostream& seekp(std::streampos pos); void close(); uint64 getWritten() const { return (_obs->written() + 7) >> 3; } protected: void writeHeader(); private: static const int BITSTREAM_TYPE; static const int BITSTREAM_FORMAT_VERSION; static const int DEFAULT_BUFFER_SIZE; static const byte COPY_BLOCK_MASK; static const byte TRANSFORMS_MASK; static const int MIN_BITSTREAM_BLOCK_SIZE; static const int MAX_BITSTREAM_BLOCK_SIZE; static const int SMALL_BLOCK_SIZE; static const int CANCEL_TASKS_ID; static const int MAX_CONCURRENCY; static inline uint32 mix32(uint32 checksum, uint32 hash, uint32 value) { checksum ^= hash * uint32(~value); checksum = (checksum << 13) | (checksum >> 19); return checksum * 5 + 0x52DCE729; } int _blockSize; int _bufferId; // index of current write buffer int _jobs; int _bufferThreshold; int _nbInputBlocks; int64 _inputSize; XXHash32* _hasher32; XXHash64* _hasher64; SliceArray** _buffers; // input & output per block short _entropyType; uint64 _transformType; DefaultOutputBitStream* _obs; atomic_int_t _initialized; atomic_int_t _closed; atomic_int_t _blockId; atomic_int_t _inputBlockId; // Counter for input blocks std::vector*> _listeners; std::vector _jobsPerTask; Context _ctx; bool _headless; #ifdef CONCURRENCY_ENABLED ThreadPool* _pool; std::vector > _futures; // Futures for async tasks std::mutex _blockMutex; std::condition_variable _blockCondition; // Drain all tasks after an asynchronous failure before releasing shared resources. void drainTasks() noexcept; #endif void processBuffer(); void submitBlock(); static void notifyListeners(std::vector*>& listeners, const Event& evt); }; inline std::streampos CompressedOutputStream::tellp() { throw std::ios_base::failure("Not supported"); } inline std::ostream& CompressedOutputStream::seekp(std::streampos) { throw std::ios_base::failure("Not supported"); } inline std::ostream& CompressedOutputStream::flush() { // NOOP: let the underlying output stream flush itself when needed return *this; } } #endif kanzi-cpp-2.6.0/src/io/IOException.hpp000066400000000000000000000026541525576310500175300ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_IOException #define knz_IOException #include #include #include "../Error.hpp" #include "../types.hpp" #include "../util/strings.hpp" namespace kanzi { class IOException : public std::runtime_error { private: int _code; public: IOException(const std::string& msg) : std::runtime_error(msg + ". Error code: " + TOSTR(Error::ERR_UNKNOWN)) { _code = Error::ERR_UNKNOWN; } IOException(const std::string& msg, int error) : std::runtime_error(msg + ". Error code: " + TOSTR(error)) { _code = error; } #if __cplusplus >= 201103L IOException(const IOException&) = default; IOException& operator=(const IOException&) = default; #endif int error() const { return _code; } ~IOException() NOEXCEPT {} }; } #endif kanzi-cpp-2.6.0/src/io/IOUtil.hpp000066400000000000000000000230371525576310500165050ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_IOUtil #define knz_IOUtil #include #include #include #include #include #include #if __cplusplus >= 201703L #include #endif #include "../types.hpp" #ifdef _MSC_VER #include "../msvc_dirent.hpp" #include #else #include #endif // Note stat64/lstat64 are deprecated on MacOS/Linux // Use _FILE_OFFSET_BITS and stat/lstat instead #ifdef _WIN32 #define STAT _stat64 #define LSTAT _stat64 #else #define STAT stat #define LSTAT lstat #endif #if defined(__MINGW32__) // Missing from stat in mingw32 const int S_IFLNK = 40960; #endif namespace kanzi { struct FileData { std::string _path; std::string _name; int64 _size; int64 _modifTime; FileData(const std::string& path, int64 size, int64 _modifTime = 0) : _size(size) , _modifTime(_modifTime) { size_t idx = path.find_last_of(PATH_SEPARATOR); if (idx != std::string::npos) { _path = path.substr(0, idx + 1); _name = path.substr(idx + 1); } else { _path = ""; _name = path; } } std::string fullPath() const { return (_path.length() == 0) ? _name : _path + _name; } }; struct FileListConfig { bool _recursive; bool _ignoreLinks; // Do not follow links bool _continueOnErrors; bool _ignoreDotFiles; // Do not process dot files }; static inline void createFileList(std::string& target, std::vector& files, const FileListConfig& cfg, std::vector& errors) { if (target.size() == 0) return; // Note: old version of Windows/Visual Studio require a trailing '/' to stat network folders ! // In this scenario, "//PC/share" does not work but "//PC/share/" does #ifndef _MSC_VER if ((target.size() > 1) && (target[target.size() - 1] == PATH_SEPARATOR)) target.resize(target.size() - 1); #endif if (cfg._ignoreDotFiles == true) { size_t idx = target.rfind(PATH_SEPARATOR); if ((idx != std::string::npos) && (idx < target.length() - 1) && (target[idx + 1] == '.')) return; } struct STAT buffer; int res = cfg._ignoreLinks ? LSTAT(target.c_str(), &buffer) : STAT(target.c_str(), &buffer); if (res != 0) { std::stringstream ss; ss << "Cannot access input file '" << target << "': " << strerror(errno); errors.push_back(ss.str()); if (cfg._continueOnErrors == false) return; } #ifdef _WIN32 if (S_ISREG(buffer.st_mode)) { #else if (S_ISREG(buffer.st_mode) || (!cfg._ignoreLinks && S_ISLNK(buffer.st_mode))) { #endif #if __cplusplus >= 201103L files.emplace_back(target, buffer.st_size, buffer.st_mtime); #else files.push_back(FileData(target, buffer.st_size, buffer.st_mtime)); #endif return; } if (!S_ISDIR(buffer.st_mode)) { // Target is neither regular file nor directory, ignore return; } if (cfg._recursive) { if (target[target.size() - 1] != PATH_SEPARATOR) target += PATH_SEPARATOR; } else { target.resize(target.size() - 1); } DIR* dir = opendir(target.c_str()); if (dir != nullptr) { const struct dirent* ent; while ((ent = readdir(dir)) != nullptr) { std::string dirName = ent->d_name; if ((dirName == ".") || (dirName == "..")) continue; std::string fullpath = target + dirName; res = cfg._ignoreLinks ? LSTAT(fullpath.c_str(), &buffer) : STAT(fullpath.c_str(), &buffer); if (res != 0) { std::stringstream ss; ss << "Cannot access input file '" << fullpath << "': " << strerror(errno); errors.push_back(ss.str()); if (cfg._continueOnErrors == false) { closedir(dir); return; } } #ifdef _WIN32 if (S_ISREG(buffer.st_mode)) { #else if (S_ISREG(buffer.st_mode) || (!cfg._ignoreLinks && S_ISLNK(buffer.st_mode))) { #endif // Target is regular file if (cfg._ignoreDotFiles == true) { size_t idx = fullpath.rfind(PATH_SEPARATOR); if ((idx != std::string::npos) && (idx < fullpath.length() - 1) && (fullpath[idx + 1] == '.')) continue; } #if __cplusplus >= 201103L files.emplace_back(fullpath, buffer.st_size, buffer.st_mtime); #else files.push_back(FileData(fullpath, buffer.st_size, buffer.st_mtime)); #endif } else if (cfg._recursive && S_ISDIR(buffer.st_mode)) { if (cfg._ignoreDotFiles == true) { size_t idx = fullpath.rfind(PATH_SEPARATOR); if ((idx != std::string::npos) && (idx < fullpath.length() - 1) && (fullpath[idx + 1] == '.')) continue; } createFileList(fullpath, files, cfg, errors); } } closedir(dir); } else { std::stringstream ss; ss << "Cannot read directory '" << target << "'"; errors.push_back(ss.str()); } } struct FileDataComparator { bool _sortBySize; bool operator() (const FileData& f1, const FileData& f2) const { // First, compare parent directory paths if (f1._path != f2._path) return f1._path < f2._path; // Then compare file sizes (decreasing order) return _sortBySize ? f1._size > f2._size : f1._name < f2._name; } }; static inline void sortFilesByPathAndSize(std::vector& files, bool sortBySize = false) { if (files.size() > 1) { FileDataComparator c = { sortBySize }; sort(files.begin(), files.end(), c); } } #if defined(WIN32) || defined(_WIN32) || defined(_WIN64) static inline int mkdirAll(const std::string& path) { bool foundDrive = false; #else static inline int mkdirAll(const std::string& path, mode_t mode = S_IRWXU | S_IRWXG | S_IROTH | S_IWOTH | S_IXOTH) { #endif int res = 0; errno = 0; // Scan path, ignoring potential PATH_SEPARATOR at position 0 for (size_t i = 1; i < path.size(); i++) { if (path[i] == PATH_SEPARATOR) { std::string curPath = path; curPath.resize(i); if (curPath.length() == 0) continue; #if defined(WIN32) || defined(_WIN32) || defined(_WIN64) //Skip if drive if ((foundDrive == false) && (curPath.length() == 2) && (curPath[1] == ':')) { foundDrive = true; continue; } #endif #if defined(_MSC_VER) res = _mkdir(curPath.c_str()); #elif defined(__MINGW32__) res = mkdir(curPath.c_str()); #else res = mkdir(curPath.c_str(), mode); #endif if ((res != 0) && (errno != EEXIST)) return errno; } } errno = 0; #if defined(_MSC_VER) res = _mkdir(path.c_str()); #elif defined(__MINGW32__) res = mkdir(path.c_str()); #else res = mkdir(path.c_str(), mode); #endif return (res == 0) ? 0 : errno; } static inline bool samePaths(const std::string& f1, const std::string& f2) { #if __cplusplus >= 201703L // Simpler and safer code with C++17 std::error_code ec; return std::filesystem::equivalent(f1, f2, ec); #else if (f1.compare(f2) == 0) return true; struct STAT buf1; int s1 = STAT(f1.c_str(), &buf1); struct STAT buf2; int s2 = STAT(f2.c_str(), &buf2); if ((s1 < 0) && (s2 < 0)) return false; if (s1 != s2) return false; if (buf1.st_dev != buf2.st_dev) return false; if (buf1.st_ino != buf2.st_ino) return false; if (buf1.st_mode != buf2.st_mode) return false; if (buf1.st_nlink != buf2.st_nlink) return false; if (buf1.st_uid != buf2.st_uid) return false; if (buf1.st_gid != buf2.st_gid) return false; if (buf1.st_rdev != buf2.st_rdev) return false; if (buf1.st_size != buf2.st_size) return false; if (buf1.st_atime != buf2.st_atime) return false; if (buf1.st_mtime != buf2.st_mtime) return false; if (buf1.st_ctime != buf2.st_ctime) return false; return true; #endif } } #endif kanzi-cpp-2.6.0/src/io/NullOutputStream.hpp000066400000000000000000000025411525576310500206440ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_NullOutputStream #define knz_NullOutputStream namespace kanzi { template > class basic_nullbuf : public std::basic_streambuf { typename traits::int_type overflow(typename traits::int_type c) { return traits::not_eof(c); } void close() {} }; template > class basic_onullstream : public std::basic_ostream { public: basic_onullstream() : std::basic_ios(&_sbuf), std::basic_ostream(&_sbuf) { this->init(&_sbuf); } private: basic_nullbuf _sbuf; }; typedef basic_onullstream NullOutputStream; } #endif kanzi-cpp-2.6.0/src/msvc_dirent.hpp000066400000000000000000000666241525576310500172570ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ /* * Dirent interface for Microsoft Visual Studio * * Copyright (C) 2006-2012 Toni Ronkko * This file is part of dirent. Dirent may be freely distributed * under the MIT license. For all details and documentation, see * https://github.com/tronkko/dirent */ #ifndef DIRENT_H #define DIRENT_H #if defined(WIN32) || defined(_WIN32) || defined(_WIN64) /* * Include windows.h without Windows Sockets 1.1 to prevent conflicts with * Windows Sockets 2.0. */ #ifndef WIN32_LEAN_AND_MEAN # define WIN32_LEAN_AND_MEAN #endif #include #include #include #include #include #include /* Indicates that d_type field is available in dirent structure */ #define _DIRENT_HAVE_D_TYPE /* Indicates that d_namlen field is available in dirent structure */ #define _DIRENT_HAVE_D_NAMLEN /* Entries missing from MSVC 6.0 */ #if !defined(FILE_ATTRIBUTE_DEVICE) # define FILE_ATTRIBUTE_DEVICE 0x40 #endif /* File type and permission flags for stat(), general mask */ #if !defined(S_IFMT) # define S_IFMT _S_IFMT #endif /* Directory bit */ #if !defined(S_IFDIR) # define S_IFDIR _S_IFDIR #endif /* Character device bit */ #if !defined(S_IFCHR) # define S_IFCHR _S_IFCHR #endif /* Pipe bit */ #if !defined(S_IFFIFO) # define S_IFFIFO _S_IFFIFO #endif /* Regular file bit */ #if !defined(S_IFREG) # define S_IFREG _S_IFREG #endif /* Read permission */ #if !defined(S_IREAD) # define S_IREAD _S_IREAD #endif /* Write permission */ #if !defined(S_IWRITE) # define S_IWRITE _S_IWRITE #endif /* Execute permission */ #if !defined(S_IEXEC) # define S_IEXEC _S_IEXEC #endif /* Pipe */ #if !defined(S_IFIFO) # define S_IFIFO _S_IFIFO #endif /* Block device */ #if !defined(S_IFBLK) # define S_IFBLK 0 #endif /* Link */ #if !defined(S_IFLNK) # define S_IFLNK 0 #endif /* Socket */ #if !defined(S_IFSOCK) # define S_IFSOCK 0 #endif /* Read user permission */ #if !defined(S_IRUSR) # define S_IRUSR S_IREAD #endif /* Write user permission */ #if !defined(S_IWUSR) # define S_IWUSR S_IWRITE #endif /* Execute user permission */ #if !defined(S_IXUSR) # define S_IXUSR S_IEXEC #endif /* Read group permission */ #if !defined(S_IRGRP) # define S_IRGRP 0x20 #endif /* Write group permission */ #if !defined(S_IWGRP) # define S_IWGRP 0x10 #endif /* Execute group permission */ #if !defined(S_IXGRP) # define S_IXGRP 0x08 #endif /* Read others permission */ #if !defined(S_IROTH) # define S_IROTH 0x04 #endif /* Write others permission */ #if !defined(S_IWOTH) # define S_IWOTH 0x02 #endif /* Execute others permission */ #if !defined(S_IXOTH) # define S_IXOTH 0x01 #endif /* Maximum length of file name */ #if !defined(PATH_MAX) # define PATH_MAX MAX_PATH #endif #if !defined(FILENAME_MAX) # define FILENAME_MAX MAX_PATH #endif #if !defined(NAME_MAX) # define NAME_MAX FILENAME_MAX #endif /* File type flags for d_type */ #define DT_UNKNOWN 0 #define DT_REG S_IFREG #define DT_DIR S_IFDIR #define DT_FIFO S_IFIFO #define DT_SOCK S_IFSOCK #define DT_CHR S_IFCHR #define DT_BLK S_IFBLK #define DT_LNK S_IFLNK /* Macros for converting between st_mode and d_type */ #define IFTODT(mode) ((mode) & S_IFMT) #define DTTOIF(type) (type) /* * File type macros. Note that block devices, sockets and links cannot be * distinguished on Windows and the macros S_ISBLK, S_ISSOCK and S_ISLNK are * only defined for compatibility. These macros should always return false * on Windows. */ #if !defined(S_ISFIFO) # define S_ISFIFO(mode) (((mode) & S_IFMT) == S_IFIFO) #endif #if !defined(S_ISDIR) # define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) #endif #if !defined(S_ISREG) # define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) #endif #if !defined(S_ISLNK) # define S_ISLNK(mode) (((mode) & S_IFMT) == S_IFLNK) #endif #if !defined(S_ISSOCK) # define S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK) #endif #if !defined(S_ISCHR) # define S_ISCHR(mode) (((mode) & S_IFMT) == S_IFCHR) #endif #if !defined(S_ISBLK) # define S_ISBLK(mode) (((mode) & S_IFMT) == S_IFBLK) #endif /* Return the exact length of the file name without zero terminator */ #define _D_EXACT_NAMLEN(p) ((p)->d_namlen) /* Return the maximum size of a file name */ #define _D_ALLOC_NAMLEN(p) ((PATH_MAX)+1) #ifdef __cplusplus extern "C" { #endif /* Wide-character version */ struct _wdirent { /* Always zero */ long d_ino; /* File position within stream */ long d_off; /* Structure size */ unsigned short d_reclen; /* Length of name without \0 */ size_t d_namlen; /* File type */ int d_type; /* File name */ wchar_t d_name[PATH_MAX+1]; }; typedef struct _wdirent _wdirent; struct _WDIR { /* Current directory entry */ struct _wdirent ent; /* Private file data */ WIN32_FIND_DATAW data; /* True if data is valid */ int cached; /* Win32 search handle */ HANDLE handle; /* Initial directory name */ wchar_t *patt; }; typedef struct _WDIR _WDIR; /* Multi-byte character version */ struct dirent { /* Always zero */ long d_ino; /* File position within stream */ long d_off; /* Structure size */ unsigned short d_reclen; /* Length of name without \0 */ size_t d_namlen; /* File type */ int d_type; /* File name */ char d_name[PATH_MAX+1]; }; typedef struct dirent dirent; struct DIR { struct dirent ent; struct _WDIR *wdirp; }; typedef struct DIR DIR; /* Dirent functions */ static DIR *opendir (const char *dirname); static _WDIR *_wopendir (const wchar_t *dirname); static struct dirent *readdir (DIR *dirp); static struct _wdirent *_wreaddir (_WDIR *dirp); static int readdir_r( DIR *dirp, struct dirent *entry, struct dirent **result); static int _wreaddir_r( _WDIR *dirp, struct _wdirent *entry, struct _wdirent **result); static int closedir (DIR *dirp); static int _wclosedir (_WDIR *dirp); static void rewinddir (DIR* dirp); static void _wrewinddir (_WDIR* dirp); static int scandir (const char *dirname, struct dirent ***namelist, int (*filter)(const struct dirent*), int (*compare)(const void *, const void *)); static int alphasort (const struct dirent **a, const struct dirent **b); static int versionsort (const struct dirent **a, const struct dirent **b); /* For compatibility with Symbian */ #define wdirent _wdirent #define WDIR _WDIR #define wopendir _wopendir #define wreaddir _wreaddir #define wclosedir _wclosedir #define wrewinddir _wrewinddir /* Internal utility functions */ static WIN32_FIND_DATAW *dirent_first (_WDIR *dirp); static WIN32_FIND_DATAW *dirent_next (_WDIR *dirp); static int dirent_mbstowcs_s( size_t *pReturnValue, wchar_t *wcstr, size_t sizeInWords, const char *mbstr, size_t count); static int dirent_wcstombs_s( size_t *pReturnValue, char *mbstr, size_t sizeInBytes, const wchar_t *wcstr, size_t count); static void dirent_set_errno (int error); /* * Open directory stream DIRNAME for read and return a pointer to the * internal working area that is used to retrieve individual directory * entries. */ static _WDIR* _wopendir( const wchar_t *dirname) { _WDIR *dirp = nullptr; int error; /* Must have directory name */ if (dirname == nullptr || dirname[0] == '\0') { dirent_set_errno (ENOENT); return nullptr; } /* Allocate new _WDIR structure */ dirp = (_WDIR*) malloc (sizeof (struct _WDIR)); if (dirp != nullptr) { DWORD n; /* Reset _WDIR structure */ dirp->handle = INVALID_HANDLE_VALUE; dirp->patt = nullptr; dirp->cached = 0; /* Compute the length of full path plus zero terminator * * Note that on WinRT there's no way to convert relative paths * into absolute paths, so just assume it is an absolute path. */ # if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_PHONE_APP) n = wcslen(dirname); # else n = GetFullPathNameW (dirname, 0, nullptr, nullptr); # endif /* Allocate room for absolute directory name and search pattern */ dirp->patt = (wchar_t*) malloc (sizeof (wchar_t) * n + 16); if (dirp->patt) { /* * Convert relative directory name to an absolute one. This * allows rewinddir() to function correctly even when current * working directory is changed between opendir() and rewinddir(). * * Note that on WinRT there's no way to convert relative paths * into absolute paths, so just assume it is an absolute path. */ # if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_PHONE_APP) wcsncpy_s(dirp->patt, n+1, dirname, n); # else n = GetFullPathNameW (dirname, n, dirp->patt, nullptr); # endif if (n > 0) { wchar_t *p; /* Append search pattern \* to the directory name */ p = dirp->patt + n; if (dirp->patt < p) { switch (p[-1]) { case '\\': case '/': case ':': /* Directory ends in path separator, e.g. c:\temp\ */ /*NOP*/; break; default: /* Directory name doesn't end in path separator */ *p++ = '\\'; } } *p++ = '*'; *p = '\0'; /* Open directory stream and retrieve the first entry */ if (dirent_first (dirp)) { /* Directory stream opened successfully */ error = 0; } else { /* Cannot retrieve first entry */ error = 1; dirent_set_errno (ENOENT); } } else { /* Cannot retrieve full path name */ dirent_set_errno (ENOENT); error = 1; } } else { /* Cannot allocate memory for search pattern */ error = 1; } } else { /* Cannot allocate _WDIR structure */ error = 1; } /* Clean up in case of error */ if (error && dirp) { _wclosedir (dirp); dirp = nullptr; } return dirp; } /* * Read next directory entry. * * Returns pointer to static directory entry which may be overwritten by * subsequent calls to _wreaddir(). */ static struct _wdirent* _wreaddir( _WDIR *dirp) { struct _wdirent *entry; /* * Read directory entry to buffer. We can safely ignore the return value * as entry will be set to nullptr in case of error. */ (void) _wreaddir_r (dirp, &dirp->ent, &entry); /* Return pointer to statically allocated directory entry */ return entry; } /* * Read next directory entry. * * Returns zero on success. If end of directory stream is reached, then sets * result to nullptr and returns zero. */ static int _wreaddir_r( _WDIR *dirp, struct _wdirent *entry, struct _wdirent **result) { WIN32_FIND_DATAW *datap; /* Read next directory entry */ datap = dirent_next (dirp); if (datap) { size_t n; DWORD attr; /* * Copy file name as wide-character string. If the file name is too * long to fit in to the destination buffer, then truncate file name * to PATH_MAX characters and zero-terminate the buffer. */ n = 0; while (n < PATH_MAX && datap->cFileName[n] != 0) { entry->d_name[n] = datap->cFileName[n]; n++; } entry->d_name[n] = 0; /* Length of file name excluding zero terminator */ entry->d_namlen = n; /* File type */ attr = datap->dwFileAttributes; if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) { entry->d_type = DT_CHR; } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) { entry->d_type = DT_DIR; } else { entry->d_type = DT_REG; } /* Reset dummy fields */ entry->d_ino = 0; entry->d_off = 0; entry->d_reclen = sizeof (struct _wdirent); /* Set result address */ *result = entry; } else { /* Return nullptr to indicate end of directory */ *result = nullptr; } return /*OK*/0; } /* * Close directory stream opened by opendir() function. This invalidates the * DIR structure as well as any directory entry read previously by * _wreaddir(). */ static int _wclosedir( _WDIR *dirp) { int ok; if (dirp) { /* Release search handle */ if (dirp->handle != INVALID_HANDLE_VALUE) { FindClose (dirp->handle); dirp->handle = INVALID_HANDLE_VALUE; } /* Release search pattern */ if (dirp->patt) { free (dirp->patt); dirp->patt = nullptr; } /* Release directory structure */ free (dirp); ok = /*success*/0; } else { /* Invalid directory stream */ dirent_set_errno (EBADF); ok = /*failure*/-1; } return ok; } /* * Rewind directory stream such that _wreaddir() returns the very first * file name again. */ static void _wrewinddir( _WDIR* dirp) { if (dirp) { /* Release existing search handle */ if (dirp->handle != INVALID_HANDLE_VALUE) { FindClose (dirp->handle); } /* Open new search handle */ dirent_first (dirp); } } /* Get first directory entry (internal) */ static WIN32_FIND_DATAW* dirent_first( _WDIR *dirp) { WIN32_FIND_DATAW *datap; /* Open directory and retrieve the first entry */ dirp->handle = FindFirstFileExW( dirp->patt, FindExInfoStandard, &dirp->data, FindExSearchNameMatch, nullptr, 0); if (dirp->handle != INVALID_HANDLE_VALUE) { /* a directory entry is now waiting in memory */ datap = &dirp->data; dirp->cached = 1; } else { /* Failed to re-open directory: no directory entry in memory */ dirp->cached = 0; datap = nullptr; } return datap; } /* * Get next directory entry (internal). * * Returns */ static WIN32_FIND_DATAW* dirent_next( _WDIR *dirp) { WIN32_FIND_DATAW *p; /* Get next directory entry */ if (dirp->cached != 0) { /* A valid directory entry already in memory */ p = &dirp->data; dirp->cached = 0; } else if (dirp->handle != INVALID_HANDLE_VALUE) { /* Get the next directory entry from stream */ if (FindNextFileW (dirp->handle, &dirp->data) != FALSE) { /* Got a file */ p = &dirp->data; } else { /* The very last entry has been processed or an error occurred */ FindClose (dirp->handle); dirp->handle = INVALID_HANDLE_VALUE; p = nullptr; } } else { /* End of directory stream reached */ p = nullptr; } return p; } /* * Open directory stream using plain old C-string. */ static DIR* opendir( const char *dirname) { struct DIR *dirp; int error; /* Must have directory name */ if (dirname == nullptr || dirname[0] == '\0') { dirent_set_errno (ENOENT); return nullptr; } /* Allocate memory for DIR structure */ dirp = (DIR*) malloc (sizeof (struct DIR)); if (dirp) { wchar_t wname[PATH_MAX + 1]; size_t n; /* Convert directory name to wide-character string */ error = dirent_mbstowcs_s( &n, wname, PATH_MAX + 1, dirname, PATH_MAX + 1); if (!error) { /* Open directory stream using wide-character name */ dirp->wdirp = _wopendir (wname); if (dirp->wdirp) { /* Directory stream opened */ error = 0; } else { /* Failed to open directory stream */ error = 1; } } else { /* * Cannot convert file name to wide-character string. This * occurs if the string contains invalid multi-byte sequences or * the output buffer is too small to contain the resulting * string. */ error = 1; } } else { /* Cannot allocate DIR structure */ error = 1; } /* Clean up in case of error */ if (error && dirp) { free (dirp); dirp = nullptr; } return dirp; } /* * Read next directory entry. */ static struct dirent* readdir( DIR *dirp) { struct dirent *entry; /* * Read directory entry to buffer. We can safely ignore the return value * as entry will be set to nullptr in case of error. */ (void) readdir_r (dirp, &dirp->ent, &entry); /* Return pointer to statically allocated directory entry */ return entry; } /* * Read next directory entry into called-allocated buffer. * * Returns zero on success. If the end of directory stream is reached, then * sets result to nullptr and returns zero. */ static int readdir_r( DIR *dirp, struct dirent *entry, struct dirent **result) { WIN32_FIND_DATAW *datap; /* Read next directory entry */ datap = dirent_next (dirp->wdirp); if (datap) { size_t n; int error; /* Attempt to convert file name to multi-byte string */ error = dirent_wcstombs_s( &n, entry->d_name, PATH_MAX + 1, datap->cFileName, PATH_MAX + 1); /* * If the file name cannot be represented by a multi-byte string, * then attempt to use old 8+3 file name. This allows traditional * Unix-code to access some file names despite of unicode * characters, although file names may seem unfamiliar to the user. * * Be ware that the code below cannot come up with a short file * name unless the file system provides one. At least * VirtualBox shared folders fail to do this. */ if (error && datap->cAlternateFileName[0] != '\0') { error = dirent_wcstombs_s( &n, entry->d_name, PATH_MAX + 1, datap->cAlternateFileName, PATH_MAX + 1); } if (!error) { DWORD attr; /* Length of file name excluding zero terminator */ entry->d_namlen = n - 1; /* File attributes */ attr = datap->dwFileAttributes; if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) { entry->d_type = DT_CHR; } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) { entry->d_type = DT_DIR; } else { entry->d_type = DT_REG; } /* Reset dummy fields */ entry->d_ino = 0; entry->d_off = 0; entry->d_reclen = sizeof (struct dirent); } else { /* * Cannot convert file name to multi-byte string so construct * an erroneous directory entry and return that. Note that * we cannot return nullptr as that would stop the processing * of directory entries completely. */ entry->d_name[0] = '?'; entry->d_name[1] = '\0'; entry->d_namlen = 1; entry->d_type = DT_UNKNOWN; entry->d_ino = 0; entry->d_off = -1; entry->d_reclen = 0; } /* Return pointer to directory entry */ *result = entry; } else { /* No more directory entries */ *result = nullptr; } return /*OK*/0; } /* * Close directory stream. */ static int closedir( DIR *dirp) { int ok; if (dirp) { /* Close wide-character directory stream */ ok = _wclosedir (dirp->wdirp); dirp->wdirp = nullptr; /* Release multi-byte character version */ free (dirp); } else { /* Invalid directory stream */ dirent_set_errno (EBADF); ok = /*failure*/-1; } return ok; } /* * Rewind directory stream to beginning. */ static void rewinddir( DIR* dirp) { /* Rewind wide-character string directory stream */ _wrewinddir (dirp->wdirp); } /* * Scan directory for entries. */ static int scandir( const char *dirname, struct dirent ***namelist, int (*filter)(const struct dirent*), int (*compare)(const void*, const void*)) { struct dirent **files = nullptr; size_t size = 0; const size_t init_size = 1; DIR *dir = nullptr; struct dirent *entry; struct dirent *tmp = nullptr; int result = 0; /* Open directory stream */ dir = opendir (dirname); if (dir) { size_t allocated = 0; /* Read directory entries to memory */ while (true) { /* Enlarge pointer table to make room for another pointer */ if (size >= allocated) { void *p; size_t num_entries; /* Compute number of entries in the enlarged pointer table */ if (size < init_size) { /* Allocate initial pointer table */ num_entries = init_size; } else { /* Double the size */ num_entries = size * 2; } /* Allocate first pointer table or enlarge existing table */ p = realloc (files, sizeof (void*) * num_entries); if (p != nullptr) { /* Got the memory */ files = (dirent**) p; allocated = num_entries; } else { /* Out of memory */ result = -1; break; } } /* Allocate room for temporary directory entry */ if (tmp == nullptr) { tmp = (struct dirent*) malloc (sizeof (struct dirent)); if (tmp == nullptr) { /* Cannot allocate temporary directory entry */ result = -1; break; } } /* Read directory entry to temporary area */ if (readdir_r (dir, tmp, &entry) == /*OK*/0) { /* Did we get an entry? */ if (entry != nullptr) { int pass; /* Determine whether to include the entry in result */ if (filter) { /* Let the filter function decide */ pass = filter (tmp); } else { /* No filter function, include everything */ pass = 1; } if (pass) { /* Store the temporary entry to pointer table */ files[size++] = tmp; tmp = nullptr; /* Keep up with the number of files */ result++; } } else { /* * End of directory stream reached => sort entries and * exit. */ qsort (files, size, sizeof (void*), compare); break; } } else { /* Error reading directory entry */ result = /*Error*/ -1; break; } } } else { /* Cannot open directory */ result = /*Error*/ -1; } /* Release temporary directory entry */ if (tmp) { free (tmp); } /* Release allocated memory on error */ if (files && (result < 0)) { for (size_t i = 0; i < size; i++) { free (files[i]); } free (files); files = nullptr; } /* Close directory stream */ if (dir) { closedir (dir); } /* Pass pointer table to caller */ if (namelist) { *namelist = files; } return result; } /* Alphabetical sorting */ static int alphasort( const struct dirent **a, const struct dirent **b) { return strcoll ((*a)->d_name, (*b)->d_name); } /* Sort versions */ static int versionsort( const struct dirent **a, const struct dirent **b) { /* FIXME: implement strverscmp and use that */ return alphasort (a, b); } /* Convert multi-byte string to wide character string */ static int dirent_mbstowcs_s( size_t *pReturnValue, wchar_t *wcstr, size_t sizeInWords, const char *mbstr, size_t count) { int error; #if defined(_MSC_VER) && _MSC_VER >= 1400 /* Microsoft Visual Studio 2005 or later */ error = mbstowcs_s (pReturnValue, wcstr, sizeInWords, mbstr, count); #else /* Older Visual Studio or non-Microsoft compiler */ size_t n; /* Convert to wide-character string (or count characters) */ n = mbstowcs (wcstr, mbstr, sizeInWords); if (!wcstr || n < count) { /* Zero-terminate output buffer */ if (wcstr && sizeInWords) { if (n >= sizeInWords) { n = sizeInWords - 1; } wcstr[n] = 0; } /* Length of resulting multi-byte string WITH zero terminator */ if (pReturnValue) { *pReturnValue = n + 1; } /* Success */ error = 0; } else { /* Could not convert string */ error = 1; } #endif return error; } /* Convert wide-character string to multi-byte string */ static int dirent_wcstombs_s( size_t *pReturnValue, char *mbstr, size_t sizeInBytes, /* max size of mbstr */ const wchar_t *wcstr, size_t count) { int error; #if defined(_MSC_VER) && _MSC_VER >= 1400 /* Microsoft Visual Studio 2005 or later */ error = wcstombs_s (pReturnValue, mbstr, sizeInBytes, wcstr, count); #else /* Older Visual Studio or non-Microsoft compiler */ size_t n; /* Convert to multi-byte string (or count the number of bytes needed) */ n = wcstombs (mbstr, wcstr, sizeInBytes); if (!mbstr || n < count) { /* Zero-terminate output buffer */ if (mbstr && sizeInBytes) { if (n >= sizeInBytes) { n = sizeInBytes - 1; } mbstr[n] = '\0'; } /* Length of resulting multi-bytes string WITH zero-terminator */ if (pReturnValue) { *pReturnValue = n + 1; } /* Success */ error = 0; } else { /* Cannot convert string */ error = 1; } #endif return error; } /* Set errno variable */ static void dirent_set_errno( int error) { #if defined(_MSC_VER) && _MSC_VER >= 1400 /* Microsoft Visual Studio 2005 and later */ _set_errno (error); #else /* Non-Microsoft compiler or older Microsoft compiler */ errno = error; #endif } #ifdef __cplusplus } #endif #endif /* WINDOWS */ #endif /*DIRENT_H*/ kanzi-cpp-2.6.0/src/test/000077500000000000000000000000001525576310500151725ustar00rootroot00000000000000kanzi-cpp-2.6.0/src/test/TestAPI.c000066400000000000000000000454661525576310500166260ustar00rootroot00000000000000#include #include #include #include #include "../api/Compressor.hpp" #include "../api/Decompressor.hpp" // Test the API (pure C code) #define ASSERT(cond, msg) \ do { \ if (!(cond)) { \ fprintf(stderr, "ASSERT FAILED: %s (%s:%d)\n", \ msg, __FILE__, __LINE__); \ exit(EXIT_FAILURE); \ } \ } while (0) #define KANZI_ERR_INVALID_PARAM 18 #ifndef PORTABLE_FMEMOPEN_H #define PORTABLE_FMEMOPEN_H #if defined(_WIN32) // Windows has no fmemopen static inline FILE* portable_fmemopen(void* buf, size_t size, const char* mode) { FILE* f = tmpfile(); if (!f) return NULL; if (buf && size > 0 && strchr(mode, 'w') == NULL) { fwrite(buf, 1, size, f); rewind(f); } return f; } #else // macOS does not have fmemopen, but has funopen // BSD may not have fmemopen // Most Linux distros do have fmemopen static inline FILE* portable_fmemopen(void* buf, size_t size, const char* mode) { (void)mode; // Only "wb" or "rb" used in tests // Simple implementation: use a temporary file and preload the buffer FILE* f = tmpfile(); if (!f) return NULL; if (buf && size > 0 && strchr(mode, 'w') == NULL) { // reading: preload buffer fwrite(buf, 1, size, f); rewind(f); } return f; } #endif #endif // Helper functions static struct cData make_params(void) { struct cData p; memset(&p, 0, sizeof(p)); strcpy(p.transform, "LZP"); strcpy(p.entropy, "FPAQ"); p.blockSize = 1024; p.jobs = 1; p.checksum = 0; p.headerless = 0; return p; } static void fill_buffer(uint8_t* buf, int size) { for (int i = 0; i < size; i++) buf[i] = (uint8_t)(i * 17 + 3); } // initCompressor invalid parameters static void test_init_invalid(void) { printf("TEST: initCompressor invalid params...\n"); struct cContext* ctx; struct cData p = make_params(); FILE* f = portable_fmemopen(NULL, 1000, "wb"); ASSERT(f != NULL, "fmemopen failed"); int rc; rc = initCompressor(NULL, f, &ctx); ASSERT(rc != 0, "init should fail on NULL params"); rc = initCompressor(&p, NULL, &ctx); ASSERT(rc != 0, "init should fail on NULL FILE"); rc = initCompressor(&p, f, NULL); ASSERT(rc != 0, "init should fail on NULL ctx"); p = make_params(); memset(p.transform, 'A', sizeof(p.transform)); rc = initCompressor(&p, f, &ctx); ASSERT(rc == KANZI_ERR_INVALID_PARAM, "init should reject unterminated transform"); p = make_params(); memset(p.entropy, 'B', sizeof(p.entropy)); rc = initCompressor(&p, f, &ctx); ASSERT(rc == KANZI_ERR_INVALID_PARAM, "init should reject unterminated entropy"); fclose(f); } // Basic initialization / disposal static void test_init_dispose(void) { printf("TEST: init + dispose...\n"); struct cData p = make_params(); struct cContext* ctx = NULL; FILE* f = portable_fmemopen(NULL, 4096, "wb"); ASSERT(f != NULL, "fmemopen failed"); ASSERT(initCompressor(&p, f, &ctx) == 0, "init failed"); ASSERT(ctx != NULL, "ctx should not be NULL"); size_t out = (size_t) -1; ASSERT(disposeCompressor(&ctx, &out) == 0, "dispose failed"); fclose(f); } // Compress small block static void test_compress_small(void) { printf("TEST: small compression...\n"); struct cData p = make_params(); struct cContext* ctx = NULL; FILE* f = portable_fmemopen(NULL, 4096, "wb"); ASSERT(f != NULL, "fmemopen failed"); ASSERT(initCompressor(&p, f, &ctx) == 0, "init failed"); uint8_t data[256]; fill_buffer(data, 256); size_t inSize = 256; size_t outSize = 0; ASSERT(compress(ctx, data, inSize, &outSize) == 0, "compress failed"); size_t flushed = 0; ASSERT(disposeCompressor(&ctx, &flushed) == 0, "dispose failed"); fclose(f); } // Oversized block static void test_compress_too_big(void) { printf("TEST: oversized block handling...\n"); struct cData p = make_params(); p.blockSize = 1024; struct cContext* ctx = NULL; FILE* f = portable_fmemopen(NULL, 4096, "wb"); ASSERT(f != NULL, "fmemopen failed"); ASSERT(initCompressor(&p, f, &ctx) == 0, "init failed"); uint8_t big[4096]; fill_buffer(big, 4096); size_t inSize = sizeof(big); size_t outSize = 0; ASSERT(compress(ctx, big, inSize, &outSize) != 0, "compress should fail on oversized input"); ASSERT(outSize == 0, "output size must be zero on error"); size_t flushed = 0; ASSERT(disposeCompressor(&ctx, &flushed) == 0, "dispose failed"); fclose(f); } // Two sequential blocks static void test_compress_two_blocks(void) { printf("TEST: two-block compression...\n"); struct cData p = make_params(); p.blockSize = 1024; struct cContext* ctx = NULL; FILE* f = portable_fmemopen(NULL, 10000, "wb"); ASSERT(f != NULL, "fmemopen failed"); ASSERT(initCompressor(&p, f, &ctx) == 0, "init failed"); uint8_t a[300], b[500]; fill_buffer(a, 300); fill_buffer(b, 500); size_t inSize, outSize; inSize = 300; outSize = 0; ASSERT(compress(ctx, a, inSize, &outSize) == 0, "block 1 failed"); ASSERT(outSize <= 0, "block 1 written bytes invalid"); inSize = 500; outSize = 0; ASSERT(compress(ctx, b, inSize, &outSize) == 0, "block 2 failed"); size_t flushed = 0; ASSERT(disposeCompressor(&ctx, &flushed) == 0, "dispose failed"); fclose(f); } static void test_compress_invalid_calls(void) { printf("TEST: compress invalid runtime params...\n"); struct cData p = make_params(); struct cContext* ctx = NULL; FILE* f = portable_fmemopen(NULL, 4096, "wb"); uint8_t data[32]; size_t outSize = 123; size_t flushed = 0; ASSERT(f != NULL, "fmemopen failed"); fill_buffer(data, sizeof(data)); ASSERT(initCompressor(&p, f, &ctx) == 0, "init failed"); ASSERT(compress(NULL, data, sizeof(data), &outSize) != 0, "compress should fail on NULL ctx"); ASSERT(compress(ctx, data, sizeof(data), NULL) != 0, "compress should fail on NULL outSize"); ASSERT(compress(ctx, NULL, 1, &outSize) != 0, "compress should fail on NULL src with non-zero size"); outSize = 7; ASSERT(compress(ctx, NULL, 0, &outSize) == 0, "compress should accept NULL src with zero size"); ASSERT(outSize == 0, "compress should not report output for zero-size input"); ASSERT(disposeCompressor(&ctx, &flushed) == 0, "dispose failed"); fclose(f); } static void test_dispose_compressor_invalid(void) { printf("TEST: disposeCompressor invalid params...\n"); struct cContext* ctx = NULL; size_t outSize = 0; ASSERT(disposeCompressor(NULL, &outSize) != 0, "dispose should fail on NULL ctx pointer"); ASSERT(disposeCompressor(&ctx, &outSize) != 0, "dispose should fail on NULL ctx"); ASSERT(disposeCompressor(&ctx, NULL) != 0, "dispose should fail on NULL outSize"); } // Utilities static void write_file(const char* path, const unsigned char* data, size_t len) { FILE* f = fopen(path, "wb"); ASSERT(f != NULL, "failed to write file"); fwrite(data, 1, len, f); fclose(f); } static void compress_to_file(const char* path, const unsigned char* data, size_t len, int headerless) { FILE* f = fopen(path, "wb"); struct cData params; struct cContext* ctx = NULL; size_t inSize = len; size_t outSize = 0; size_t flushed = 0; ASSERT(f != NULL, "failed to open output file"); memset(¶ms, 0, sizeof(params)); strcpy(params.transform, "LZ"); strcpy(params.entropy, "ANS0"); params.blockSize = 1 << 15; params.jobs = 1; params.checksum = 0; params.headerless = headerless; ASSERT(initCompressor(¶ms, f, &ctx) == 0, "failed to init compressor"); ASSERT(compress(ctx, data, inSize, &outSize) == 0, "failed to compress sample"); ASSERT(disposeCompressor(&ctx, &flushed) == 0, "failed to dispose compressor"); fclose(f); } static void test_init_decompressor_invalid(void) { printf("TEST: initDecompressor invalid params...\n"); struct dContext* ctx = NULL; struct dData p; FILE* f = portable_fmemopen(NULL, 4096, "rb"); int rc; ASSERT(f != NULL, "fmemopen failed"); memset(&p, 0, sizeof(p)); p.bufferSize = 1024; p.jobs = 1; p.headerless = 0; rc = initDecompressor(NULL, f, &ctx); ASSERT(rc != 0, "initDecompressor should fail on NULL params"); rc = initDecompressor(&p, NULL, &ctx); ASSERT(rc != 0, "initDecompressor should fail on NULL FILE"); rc = initDecompressor(&p, f, NULL); ASSERT(rc != 0, "initDecompressor should fail on NULL ctx"); p.bufferSize = ((size_t)2 * 1024 * 1024 * 1024) + 1; rc = initDecompressor(&p, f, &ctx); ASSERT(rc != 0, "initDecompressor should fail on huge buffer"); memset(&p, 0, sizeof(p)); p.bufferSize = 1024; p.jobs = 1; p.headerless = 1; p.blockSize = 1024; strcpy(p.entropy, "ANS0"); memset(p.transform, 'A', sizeof(p.transform)); rc = initDecompressor(&p, f, &ctx); ASSERT(rc == KANZI_ERR_INVALID_PARAM, "initDecompressor should reject unterminated transform"); memset(&p, 0, sizeof(p)); p.bufferSize = 1024; p.jobs = 1; p.headerless = 1; p.blockSize = 1024; strcpy(p.transform, "LZ"); memset(p.entropy, 'B', sizeof(p.entropy)); rc = initDecompressor(&p, f, &ctx); ASSERT(rc == KANZI_ERR_INVALID_PARAM, "initDecompressor should reject unterminated entropy"); fclose(f); } static void test_decompress_invalid_calls(void) { printf("TEST: decompress invalid runtime params...\n"); const unsigned char input[] = "runtime validation sample"; const char* f_name = "tmp_api_invalid.bin"; FILE* fdec; struct dData params; struct dContext* ctx = NULL; unsigned char out[64]; size_t inSize = 0; size_t outSize = sizeof(out); compress_to_file(f_name, input, strlen((const char*)input), 0); fdec = fopen(f_name, "rb"); ASSERT(fdec != NULL, "failed to open file for reading"); memset(¶ms, 0, sizeof(params)); params.bufferSize = sizeof(out); params.jobs = 1; params.headerless = 0; ASSERT(initDecompressor(¶ms, fdec, &ctx) == 0, "failed to init decompressor"); ASSERT(decompress(NULL, out, &inSize, &outSize) != 0, "decompress should fail on NULL ctx"); outSize = params.bufferSize + 1; ASSERT(decompress(ctx, out, &inSize, &outSize) != 0, "decompress should fail when output buffer exceeds configured size"); outSize = 1; ASSERT(decompress(ctx, NULL, &inSize, &outSize) != 0, "decompress should fail on NULL dst"); outSize = 0; inSize = 123; ASSERT(decompress(ctx, out, &inSize, &outSize) == 0, "decompress should accept zero-sized output"); ASSERT(disposeDecompressor(&ctx) == 0, "failed to dispose decompressor"); fclose(fdec); remove(f_name); } static void test_dispose_decompressor_invalid(void) { printf("TEST: disposeDecompressor invalid params...\n"); struct dContext* ctx = NULL; ASSERT(disposeDecompressor(NULL) != 0, "disposeDecompressor should fail on NULL ctx pointer"); ASSERT(disposeDecompressor(&ctx) != 0, "disposeDecompressor should fail on NULL ctx"); } // Simple decompression of known data static void test_basic_decompression(void) { printf("TEST: basic decompression...\n"); const char* input = "Hello Kanzi! Hello Compression!"; const size_t in_len = strlen(input); // Step 1: Compress to temporary file const char* f_name = "tmp_comp.bin"; FILE* fcomp = fopen(f_name, "wb"); ASSERT(fcomp != NULL, "failed to open file"); struct cData cparams; memset(&cparams, 0, sizeof(cparams)); strcpy(cparams.transform, "LZ"); strcpy(cparams.entropy, "ANS0"); cparams.blockSize = 1 << 16; cparams.jobs = 1; cparams.checksum = 32; cparams.headerless = 0; struct cContext* cctx = NULL; ASSERT(initCompressor(&cparams, fcomp, &cctx) == 0, "failed to init compressor"); size_t inSize = in_len; size_t outSize = 0; ASSERT(compress(cctx, (const unsigned char*)input, inSize, &outSize) == 0, "failed to compress data"); size_t flushed = 0; ASSERT(disposeCompressor(&cctx, &flushed) == 0, "failed to dispose compressor"); fclose(fcomp); // Step 2: Decompress from temporary file FILE* fdec = fopen(f_name, "rb"); ASSERT(fdec != NULL, "failed to open file for reading"); struct dData dparams; memset(&dparams, 0, sizeof(dparams)); dparams.bufferSize = 1 << 16; dparams.jobs = 1; dparams.headerless = 0; struct dContext* dctx = NULL; ASSERT(initDecompressor(&dparams, fdec, &dctx) == 0, "failed to init decompressor"); unsigned char outbuf[1024]; size_t readComp = 0; size_t produced = sizeof(outbuf); ASSERT(decompress(dctx, outbuf, &readComp, &produced) == 0, "failed to decompress data"); // Step 3: Validate output ASSERT(produced == in_len, "failed to decompress data: invalid data size"); ASSERT(memcmp(outbuf, input, in_len) == 0, "failed to decompress data: data differ from original"); ASSERT(disposeDecompressor(&dctx) == 0, "failed to dispose decompressor"); fclose(fdec); remove(f_name); } // Decompress much larger data (multi-block) static void test_large_multi_block(void) { printf("TEST: large multi blocks\n"); size_t size = 2 * 1024 * 1024; // 2MB unsigned char* data = (unsigned char*)malloc(size); ASSERT(data != NULL, "failed to allocate buffer memory"); for (size_t i = 0; i < size; i++) data[i] = (unsigned char)(i * 7); const char* f_name = "tmp_large_input.bin"; write_file(f_name, data, size); // Compress const char* fcomp_name = "tmp_large_comp.bin"; FILE* fcomp = fopen(fcomp_name, "wb"); ASSERT(fcomp, "failed to open file for writing"); struct cData cparams; memset(&cparams, 0, sizeof(cparams)); strcpy(cparams.transform, "LZ"); strcpy(cparams.entropy, "FPAQ"); cparams.blockSize = 256 * 1024; cparams.jobs = 1; cparams.checksum = 64; cparams.headerless = 0; struct cContext* cctx = NULL; ASSERT(initCompressor(&cparams, fcomp, &cctx) == 0, "failed to init compressor"); size_t remaining = size; unsigned char* p = data; while (remaining > 0) { size_t chunk = ((remaining > cparams.blockSize) ? cparams.blockSize : remaining); size_t inSize = chunk; size_t outSize = 0; ASSERT(compress(cctx, p, inSize, &outSize) == 0, "failed to compress data"); p += chunk; remaining -= chunk; } size_t flushed = 0; ASSERT(disposeCompressor(&cctx, &flushed) == 0, "failed to dispose compressor"); fclose(fcomp); // Decompress FILE* fdec = fopen(fcomp_name, "rb"); ASSERT(fdec, "failed to open file for reading"); struct dData dparams; memset(&dparams, 0, sizeof(dparams)); dparams.bufferSize = 256 * 1024; dparams.jobs = 1; dparams.headerless = 0; struct dContext* dctx = NULL; ASSERT(initDecompressor(&dparams, fdec, &dctx) == 0, "failed to init decompressor"); unsigned char* out = (unsigned char*)malloc(size); ASSERT(out, "failed to allocate buffer memory"); size_t totalOut = 0; while (1) { size_t inBytes = 0; size_t outBytes = dparams.bufferSize; int r = decompress(dctx, out + totalOut, &inBytes, &outBytes); if (r != 0) break; // expected EOF if (outBytes == 0) break; totalOut += outBytes; } ASSERT(totalOut == size, "failed to decompress: invalid data size"); ASSERT(memcmp(out, data, size) == 0, "failed to decompress: data differ from original"); disposeDecompressor(&dctx); fclose(fdec); remove(f_name); remove(fcomp_name); free(out); free(data); } // Headerless mode static void test_headerless(void) { printf("TEST: headerless\n"); const char* input = "HEADERLESS MODE IS ACTIVE"; const char* f_name = "tmp_hl_input.bin"; write_file(f_name, (const unsigned char*)input, strlen(input)); // Compress with headerless = 1 const char* fcomp_name = "tmp_hl_comp.bin"; FILE* fcomp = fopen(fcomp_name, "wb"); ASSERT(fcomp, "failed to open file for writing"); struct cData cparams; memset(&cparams, 0, sizeof(cparams)); strcpy(cparams.transform, "LZ"); strcpy(cparams.entropy, "ANS0"); cparams.blockSize = 1 << 15; cparams.jobs = 1; cparams.checksum = 0; cparams.headerless = 1; struct cContext* cctx = NULL; ASSERT(initCompressor(&cparams, fcomp, &cctx) == 0, "failed to init compressor"); size_t inSize = strlen(input); size_t outSize = 0; ASSERT(compress(cctx, (const unsigned char*)input, inSize, &outSize) == 0, "failed to compress data"); size_t flushed = 0; ASSERT(disposeCompressor(&cctx, &flushed) == 0, "failed to dispose compressor"); fclose(fcomp); // Decompress with headerless = 1 FILE* fdec = fopen(fcomp_name, "rb"); ASSERT(fdec, "failed to open file for reading"); struct dData dparams; memset(&dparams, 0, sizeof(dparams)); dparams.bufferSize = 1 << 15; dparams.jobs = 1; dparams.headerless = 1; strcpy(dparams.transform, "LZ"); strcpy(dparams.entropy, "ANS0"); dparams.blockSize = 1 << 15; dparams.originalSize = strlen(input); dparams.checksum = 0; dparams.bsVersion = 7; struct dContext* dctx = NULL; ASSERT(initDecompressor(&dparams, fdec, &dctx) == 0, "failed to init decompressor"); unsigned char outbuf[256]; size_t inBytes = 0; size_t outBytes = sizeof(outbuf); ASSERT(decompress(dctx, outbuf, &inBytes, &outBytes) == 0, "failed to decompress data"); ASSERT(outBytes == strlen(input), "failed to decompress data: wrong data size"); ASSERT(memcmp(outbuf, input, strlen(input)) == 0, "failed to decompress data: data differ from original"); disposeDecompressor(&dctx); fclose(fdec); remove(f_name); remove(fcomp_name); } int main(void) { // Compressor test_init_invalid(); test_init_dispose(); test_compress_small(); test_compress_too_big(); test_compress_two_blocks(); test_compress_invalid_calls(); test_dispose_compressor_invalid(); // Decompressor test_init_decompressor_invalid(); test_decompress_invalid_calls(); test_dispose_decompressor_invalid(); test_basic_decompression(); test_large_multi_block(); test_headerless(); printf("All C API tests passed.\n"); return 0; } kanzi-cpp-2.6.0/src/test/TestBWT.cpp000066400000000000000000000252431525576310500172000ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include "../types.hpp" #include "../util/strings.hpp" #include "../transform/BWT.hpp" #include "../transform/BWTS.hpp" using namespace std; using namespace kanzi; int testBWTCorrectness(bool isBWT) { // Test behavior cout << endl << endl << (isBWT ? "BWT" : "BWTS") << " Correctness test" << endl; srand((uint)time(nullptr)); int res = 0; kanzi::byte* pBuf = new kanzi::byte[8 * 1024 * 1024]; kanzi::byte* buf1 = pBuf; for (int ii = 1; ii <= 20; ii++) { int size = 128; if (ii == 1) { string str("mississippi"); const char* val2 = str.c_str(); cout << val2 << endl; size = int(str.length()); memcpy(buf1, &val2[0], size); } else if (ii == 2) { string str("3.14159265358979323846264338327950288419716939937510"); const char* val2 = str.c_str(); size = int(str.length()); memcpy(buf1, &val2[0], size); } else if (ii == 3) { string str("SIX.MIXED.PIXIES.SIFT.SIXTY.PIXIE.DUST.BOXES"); const char* val2 = str.c_str(); size = int(str.length()); memcpy(buf1, &val2[0], size); } else if (ii < 20) { for (int i = 0; i < size; i++) buf1[i] = kanzi::byte(65 + (rand() % (4 * ii))); } else { size = 8*1024*1024; for (int i = 0; i < size; i++) buf1[i] = kanzi::byte(i); } Transform* tf; if (isBWT) { tf = new BWT(); } else { tf = new BWTS(); } kanzi::byte* input = &buf1[0]; kanzi::byte* transform = new kanzi::byte[size]; kanzi::byte* reverse = new kanzi::byte[size]; cout << endl << "Test " << ii << endl; if (size < 512) { cout << "Input : "; for (int i = 0; i < size; i++) cout << char(input[i]); } SliceArray ia1(input, size, 0); SliceArray ia2(transform, size, 0); tf->forward(ia1, ia2, size); if (size < 512) { cout << endl << "Encoded : "; for (int i = 0; i < size; i++) cout << char(transform[i]); cout << " "; } if (isBWT) { BWT* bwt = (BWT*) tf; int chunks = BWT::getBWTChunks(size); int* pi = new int[chunks]; for (int i=0; igetPrimaryIndex(i); cout << "(Primary index=" << pi[i] << ")" << endl; } delete tf; tf = new BWT(); bwt = (BWT*) tf; for (int i=0; isetPrimaryIndex(i, pi[i]); } delete[] pi; } else { delete tf; tf = new BWTS(); cout << endl; } SliceArray ia3(reverse, size, 0); ia2._index = 0; tf->inverse(ia2, ia3, size); int idx = -1; if (size < 512) { cout << "Reverse : "; for (int i = 0; i < size; i++) cout << char(reverse[i]); } for (int j = 0; j < size; j++) { if (input[j] != reverse[j]) { idx = j; res = 1; break; } } cout << endl; if (idx == -1) cout << "Identical" << endl; else cout << "Different at index " << idx << " (" << int(input[idx]) << "<->" << int(reverse[idx]) << ")" << endl; delete tf; delete[] transform; delete[] reverse; } delete[] pBuf; return res; } int testBWTSpeed(bool isBWT, int iter, bool isSmallSize) { // Test speed int size = isSmallSize ? 256 * 1024 : 10 * 1024 * 1024; int res = 0; cout << endl << endl << (isBWT ? "BWT" : "BWTS") << " Speed test" << endl; cout << "Iterations: " << iter << endl; cout << "Transform size: " << size << endl; srand(uint(time(nullptr))); for (int jj = 0; jj < 3; jj++) { kanzi::byte* input = isSmallSize ? new kanzi::byte[256 * 1024] : new kanzi::byte[10 * 1024 * 1024]; kanzi::byte* output = isSmallSize ? new kanzi::byte[256 * 1024] : new kanzi::byte[10 * 1024 * 1024]; kanzi::byte* reverse = isSmallSize ? new kanzi::byte[256 * 1024] : new kanzi::byte[10 * 1024 * 1024]; SliceArray ia1(input, size, 0); SliceArray ia2(output, size, 0); SliceArray ia3(reverse, size, 0); double delta1 = 0, delta2 = 0; Transform* tf = nullptr; Transform* ti = nullptr; const int chunks = BWT::getBWTChunks(size); int pi[8]; for (int ii = 0; ii < iter; ii++) { if (isBWT) { tf = new BWT(); } else { tf = new BWTS(); } for (int i = 0; i < size; i++) { input[i] = kanzi::byte(1 + (rand() % 255)); } clock_t before1 = clock(); ia1._index = 0; ia2._index = 0; tf->forward(ia1, ia2, size); clock_t after1 = clock(); delta1 += (after1 - before1); if (isBWT) { BWT* bwt = (BWT*)tf; for (int i = 0; i < chunks; i++) pi[i] = bwt->getPrimaryIndex(i); } delete tf; clock_t before2 = clock(); ia2._index = 0; ia3._index = 0; if (isBWT) { ti = new BWT(); BWT* bwt = (BWT*)ti; for (int i = 0; i < chunks; i++) bwt->setPrimaryIndex(i, pi[i]); } else { ti = new BWTS(); } ti->inverse(ia2, ia3, size); clock_t after2 = clock(); delta2 += (after2 - before2); delete ti; // Sanity check for (int i = 0; i < size; i++) { if (input[i] != reverse[i]) { cout << "Failure at index " << i << " (" << int(input[i]) << "<->" << int(reverse[i]) << ")" << endl; res = 1; break; } } } delete[] input; delete[] output; delete[] reverse; // KB = 1000, KiB = 1024 double prod = double(iter) * double(size); double b2KiB = double(1) / double(1024); double d1_sec = double(delta1) / CLOCKS_PER_SEC; double d2_sec = double(delta2) / CLOCKS_PER_SEC; cout << "Forward transform [ms] : " << int(d1_sec * 1000) << endl; cout << "Throughput [KiB/s] : " << int(prod * b2KiB / d1_sec) << endl; cout << "Reverse transform [ms] : " << int(d2_sec * 1000) << endl; cout << "Throughput [KiB/s] : " << int(prod * b2KiB / d2_sec) << endl; cout << endl; } return res; } int testBWTInvalidSecondaryIndex() { cout << endl << endl << "BWT invalid secondary index test" << endl; int res = 0; const int largeSize = 1024; const int smallSize = 256; BWT encoder; BWT decoder; kanzi::byte* largeInput = new kanzi::byte[largeSize]; kanzi::byte* largeTransform = new kanzi::byte[largeSize]; kanzi::byte* largeReverse = new kanzi::byte[largeSize]; kanzi::byte* smallInput = new kanzi::byte[smallSize]; kanzi::byte* smallTransform = new kanzi::byte[smallSize]; kanzi::byte* smallReverse = new kanzi::byte[smallSize]; for (int i = 0; i < largeSize; i++) largeInput[i] = kanzi::byte(i); for (int i = 0; i < smallSize; i++) smallInput[i] = kanzi::byte((i * 17) & 0xFF); SliceArray largeIa1(largeInput, largeSize, 0); SliceArray largeIa2(largeTransform, largeSize, 0); SliceArray largeIa3(largeReverse, largeSize, 0); if (encoder.forward(largeIa1, largeIa2, largeSize) == false) { res = 1; } else { for (int i = 0; i < BWT::getBWTChunks(largeSize); i++) decoder.setPrimaryIndex(i, encoder.getPrimaryIndex(i)); largeIa2._index = 0; if (decoder.inverse(largeIa2, largeIa3, largeSize) == false) res = 1; } SliceArray smallIa1(smallInput, smallSize, 0); SliceArray smallIa2(smallTransform, smallSize, 0); SliceArray smallIa3(smallReverse, smallSize, 0); if (res == 0) { largeIa1._index = 0; smallIa1._index = 0; if (encoder.forward(smallIa1, smallIa2, smallSize) == false) { res = 1; } else { const int chunks = BWT::getBWTChunks(smallSize); for (int i = 0; i < chunks; i++) decoder.setPrimaryIndex(i, encoder.getPrimaryIndex(i)); decoder.setPrimaryIndex(1, smallSize + 1); smallIa2._index = 0; if (decoder.inverse(smallIa2, smallIa3, smallSize) == true) res = 1; } } delete[] largeInput; delete[] largeTransform; delete[] largeReverse; delete[] smallInput; delete[] smallTransform; delete[] smallReverse; cout << (res == 0 ? "OK" : "Failed") << endl; return res; } #ifdef __GNUG__ int main(int argc, const char* argv[]) #else int TestBWT_main(int argc, const char* argv[]) #endif { bool doPerf = true; if (argc > 1) { string str = argv[1]; transform(str.begin(), str.end(), str.begin(), safeToUpper); doPerf = str != "-NOPERF"; } int res = 0; res |= testBWTCorrectness(true); res |= testBWTCorrectness(false); res |= testBWTInvalidSecondaryIndex(); if (doPerf) { res |= testBWTSpeed(true, 200, true); // test MergeTPSI inverse res |= testBWTSpeed(true, 5, false); // test BiPSIv2 inverse res |= testBWTSpeed(false, 200, true); } return res; } kanzi-cpp-2.6.0/src/test/TestCompressedStream.cpp000066400000000000000000000233451525576310500220250ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include "../io/CompressedInputStream.hpp" #include "../io/CompressedOutputStream.hpp" #include "../io/IOException.hpp" using namespace std; using namespace kanzi; uint64 compress1(kanzi::byte block[], uint length) { cout << "Test - Regular (RLT+TEXT&HUFFMAN)" << endl; uint blockSize = (length / (1 + (rand() & 3))) & -16; kanzi::byte* buf = new kanzi::byte[length]; memcpy(&buf[0], &block[0], size_t(length)); stringbuf buffer; iostream ios(&buffer); CompressedOutputStream* cos = new CompressedOutputStream(ios, 1, "HUFFMAN", "RLT+TEXT", blockSize); cos->write((const char*)block, length); cos->close(); uint64 written = cos->getWritten(); ios.seekg(0); memset(&block[0], 0, size_t(length)); CompressedInputStream* cis = new CompressedInputStream(ios, 1); while (true) { cis->read((char*)block, length); if (cis->gcount() != streamsize(length)) break; } cis->close(); uint64 read = cis->getRead(); delete cos; delete cis; uint64 res = (memcmp(&buf[0], &block[0], length) != 0) ? 3 : read ^ written; delete[] buf; return res; } uint64 compress2(kanzi::byte block[], uint length) { int jobs; srand((uint)time(nullptr)); int checksum = 32 * min(rand() & 3, 2); uint blockSize = (length / (1 + (rand() & 3))) & -16; #ifdef CONCURRENCY_ENABLED jobs = 1 + (rand() & 3); cout << "Test - " << jobs << " job(s) - "; #else jobs = 1; cout << "Test - "; #endif if (checksum == 0) cout << "no checksum"; else cout << checksum << " bits checksum"; cout << " (LZX&ANS0)" << endl; kanzi::byte* buf = new kanzi::byte[length]; memcpy(&buf[0], &block[0], size_t(length)); stringbuf buffer; iostream ios(&buffer); CompressedOutputStream* cos = new CompressedOutputStream(ios, jobs, "ANS0", "LZX", blockSize, checksum); cos->write((const char*)block, length); cos->close(); uint64 written = cos->getWritten(); ios.seekg(0); memset(&block[0], 0, size_t(length)); CompressedInputStream* cis = new CompressedInputStream(ios, jobs); while (true) { cis->read((char*)block, length); if (cis->gcount() != streamsize(length)) break; } cis->close(); uint64 read = cis->getRead(); delete cos; delete cis; uint64 res = (memcmp(&buf[0], &block[0], length) != 0) ? 3 : read ^ written; delete[] buf; return res; } uint64 compress3(kanzi::byte block[], uint length) { cout << "Test - incompressible data (LZP+RLT&FPAQ)" << endl; uint blockSize = (length / (1 + (rand() & 3))) & -16; kanzi::byte* buf = new kanzi::byte[length]; memcpy(&buf[0], &block[0], size_t(length)); stringbuf buffer; iostream ios(&buffer); CompressedOutputStream* cos = new CompressedOutputStream(ios, 1, "FPAQ", "LZP+ZRLT", blockSize, 32); cos->write((const char*)block, length); cos->close(); uint64 written = cos->getWritten(); ios.seekg(0); memset(&block[0], 0, size_t(length)); CompressedInputStream* cis = new CompressedInputStream(ios, 1); while (true) { cis->read((char*)block, length); if (cis->gcount() != streamsize(length)) break; } cis->close(); uint64 read = cis->getRead(); delete cos; delete cis; uint64 res = (memcmp(&buf[0], &block[0], length) != 0) ? 3 : read ^ written; delete[] buf; return res; } uint64 compress4(kanzi::byte block[], uint length) { CompressedOutputStream* cos = nullptr; uint64 res; try { cout << "Test - write after close (TEXT&HUFFMAN)" << endl; stringbuf buffer; iostream os(&buffer); cos = new CompressedOutputStream(os, 1, "HUFFMAN", "TEXT"); cos->write((const char*)block, length); cos->close(); cos->put(char(0)); cout << "Failure: no exception thrown in write after close" << endl; res = 1; } catch (const ios_base::failure& e) { cout << "OK, Exception " << e.what() << endl; res = 0; } delete cos; return res; } uint64 compress5(kanzi::byte block[], uint length) { CompressedOutputStream* cos = nullptr; CompressedInputStream* cis = nullptr; uint64 res; try { cout << "Test - read after close (TEXT&HUFFMAN)" << endl; stringbuf buffer; iostream ios(&buffer); cos = new CompressedOutputStream(ios, 1, "HUFFMAN", "TEXT"); cos->write((const char*)block, length); cos->close(); ios.seekg(0); cis = new CompressedInputStream(ios, 1); while (true) { cis->read((char*)block, length); if (cis->gcount() != streamsize(length)) break; } cis->close(); cis->get(); cout << "Failure: no exception thrown in read after close" << endl; res = 1; } catch (const ios_base::failure& e) { cout << "OK, Exception " << e.what() << endl; res = 0; } delete cos; delete cis; return res; } uint64 compress6(kanzi::byte block[]) { CompressedOutputStream* cos = nullptr; CompressedInputStream* cis = nullptr; uint64 res; try { cout << "Test - large read request does not overflow streamsize" << endl; stringbuf buffer; iostream ios(&buffer); cos = new CompressedOutputStream(ios, 1, "HUFFMAN", "TEXT"); cos->write((const char*)block, 1); cos->close(); ios.seekg(0); cis = new CompressedInputStream(ios, 1); char decoded[1] = { 0 }; const int64 requested64 = int64((numeric_limits::max)()) + 1; const streamsize requested = (int64((numeric_limits::max)()) >= requested64) ? streamsize(requested64) : (numeric_limits::max)(); cis->read(decoded, requested); if ((cis->gcount() != 1) || (decoded[0] != char(block[0]))) { cout << "Failure: invalid read result for large request" << endl; res = 1; } else { res = 0; } } catch (const ios_base::failure& e) { cout << "Failure: unexpected exception " << e.what() << endl; res = 1; } delete cos; delete cis; return res; } int testCorrectness(int, const char*[]) { // Test correctness cout << "Correctness Test" << endl; const int length0 = 65536; kanzi::byte* incompressible = new kanzi::byte[length0 << 6]; kanzi::byte* values = new kanzi::byte[length0 << 6]; bool res = true; srand((uint)time(nullptr)); for (int test = 1; test <= 50; test++) { const int length = length0 << (test % 7); for (int i = 0; i < length; i++) { incompressible[i] = kanzi::byte(rand()); values[i] = kanzi::byte(rand() % (4 * test + 1)); } cout << endl; cout << "Iteration " << test << " (size " << length << ")" << endl; uint64 cres; cres = compress1(values, length); cout << ((cres == 0) ? "Success" : "Failure") << endl; res &= (cres == 0); cres = compress2(values, length); cout << ((cres == 0) ? "Success" : "Failure") << endl; res &= (cres == 0); cres = compress3(incompressible, length); cout << ((cres == 0) ? "Success" : "Failure") << endl; res &= (cres == 0); if (test == 1) { cres = compress4(values, length); cout << ((cres == 0) ? "Success" : "Failure") << endl; res &= (cres == 0); cres = compress5(values, length); cout << ((cres == 0) ? "Success" : "Failure") << endl; res &= (cres == 0); cres = compress6(values); cout << ((cres == 0) ? "Success" : "Failure") << endl; res &= (cres == 0); } } delete[] incompressible; delete[] values; return (res == true) ? 0 : 1; } void testSeek(string name) { #if !defined(_MSC_VER) || _MSC_VER > 1500 ifstream ifs; ifs.open(name.c_str(), ios::binary|ios::in); char buf[1024]; int64 pos1 = 0; int64 pos2 = 0; CompressedInputStream cis(ifs, 1); cis.read((char*)buf, 100); for (int i = 0; i < 100; i++) cout << buf[i]; cout << endl << endl; // Block positions for enwik8 compressed with L5 & version 2.4 // To be updated. int64 positions[4] = { 17729391, 26695626, 8843019, 192 }; for (int i = 0; i< 4; i++) { pos1 = positions[i]; cis.seek(pos1); pos2 = cis.tell(); cout << pos1 << " / " << pos2 << endl; cis.read(buf, 100); for (int j = 0; j < 100; j++) cout << buf[j]; cout << endl << endl; } cis.close(); ifs.close(); #endif } #ifdef __GNUG__ int main(int argc, const char* argv[]) #else int TestCompressedStream_main(int argc, const char* argv[]) #endif { try { //testSeek("/tmp/enwik8.knz"); return testCorrectness(argc, argv); } catch (const IOException& e) { cout << "Exception: " << e.what() << endl; return e.error(); } catch (const runtime_error& e) { cout << "Exception: " << e.what() << endl; return 255; } } kanzi-cpp-2.6.0/src/test/TestDefaultBitStream.cpp000066400000000000000000000561431525576310500217460ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include #include #include "../util/strings.hpp" #include "../bitstream/DebugOutputBitStream.hpp" #include "../bitstream/DefaultInputBitStream.hpp" #include "../bitstream/DefaultOutputBitStream.hpp" #include "../BitStreamException.hpp" #include "../io/IOUtil.hpp" #include "../io/IOException.hpp" using namespace std; using namespace kanzi; int testBitStreamCorrectnessAligned1() { // Test correctness (kanzi::byte aligned) cout << "Correctness Test - write long - byte aligned" << endl; const int length = 100; int* values = new int[length]; int res = 0; srand((uint)time(nullptr)); cout << "\nInitial" << endl; // Check correctness of read() and written() for (int t = 1; t <= 32; t++) { stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios, 16384); cout << endl; obs.writeBits(0x01234567L, t); cout << "Written (before close): " << obs.written() << endl; obs.close(); cout << "Written (after close): " << obs.written() << endl; ios.rdbuf()->pubseekpos(0); DefaultInputBitStream ibs(ios, 16384); ibs.readBits(t); cout << ((ibs.read() == uint64(t)) ? "OK" : "KO") << endl; cout << "Read (before close): " << ibs.read() << endl; ibs.close(); cout << "Read (after close): " << ibs.read() << endl; } for (int test = 1; test <= 10; test++) { stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios, 16384); DebugOutputBitStream dbs(obs, cout); dbs.showByte(true); for (int i = 0; i < length; i++) { values[i] = rand(); cout << values[i] << " "; if ((i % 20) == 19) cout << endl; } cout << endl << endl; for (int i = 0; i < length; i++) { dbs.writeBits(values[i], 32); } // Close first to force flush() dbs.close(); ios.rdbuf()->pubseekpos(0); istringstream is; char* cvalues = new char[4 * length]; for (int i = 0; i < length; i++) { cvalues[4 * i] = (values[i] >> 24) & 0xFF; cvalues[4 * i + 1] = (values[i] >> 16) & 0xFF; cvalues[4 * i + 2] = (values[i] >> 8) & 0xFF; cvalues[4 * i + 3] = values[i] & 0xFF; } is.read(cvalues, length); DefaultInputBitStream ibs(ios, 16384); cout << endl << endl << "Read:" << endl; bool ok = true; for (int i = 0; i < length; i++) { int x = (int)ibs.readBits(32); cout << x; cout << ((x == values[i]) ? " " : "* "); ok &= (x == values[i]); if ((i % 20) == 19) cout << endl; } delete[] cvalues; ibs.close(); res = (ok == true) ? 0 : 1; cout << endl; cout << endl << "Bits written: " << dbs.written() << endl; cout << endl << "Bits read: " << ibs.read() << endl; cout << endl << "\n" << (ok ? "Success" : "Failure") << endl; cout << endl; cout << endl; } delete[] values; return res; } int testBitStreamCorrectnessMisaligned1() { // Test correctness (not kanzi::byte aligned) cout << "Correctness Test - write long - not byte aligned" << endl; const int length = 100; int* values = new int[length]; int res = 0; srand((uint)time(nullptr)); cout << "\nInitial" << endl; // Check correctness of read() and written() for (int t = 1; t <= 32; t++) { stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios, 16384); cout << endl; obs.writeBit(1); obs.writeBits(0x01234567L, t); cout << "Written (before close): " << obs.written() << endl; obs.close(); cout << "Written (after close): " << obs.written() << endl; ios.rdbuf()->pubseekpos(0); DefaultInputBitStream ibs(ios, 16384); ibs.readBit(); ibs.readBits(t); cout << ((ibs.read() == uint64(t + 1)) ? "OK" : "KO") << endl; cout << "Read (before close): " << ibs.read() << endl; ibs.close(); cout << "Read (after close): " << ibs.read() << endl; } for (int test = 1; test <= 10; test++) { stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios, 16384); DebugOutputBitStream dbs(obs, cout); dbs.showByte(true); for (int i = 0; i < length; i++) { values[i] = rand(); const int mask = (1 << (1 + (i % 30))) - 1; values[i] &= mask; cout << values[i] << " "; if ((i % 20) == 19) cout << endl; } cout << endl << endl; for (int i = 0; i < length; i++) { dbs.writeBits(values[i], 1 + (i % 30)); } // Close first to force flush() dbs.close(); if (test == 10) { try { cout << "\nTrying to write to closed stream" << endl; dbs.writeBit(1); } catch (const BitStreamException& e) { cout << "\nException: " << e.what() << endl; } } ios.rdbuf()->pubseekpos(0); istringstream is; char* cvalues = new char[4 * length]; for (int i = 0; i < length; i++) { cvalues[4 * i] = (values[i] >> 24) & 0xFF; cvalues[4 * i + 1] = (values[i] >> 16) & 0xFF; cvalues[4 * i + 2] = (values[i] >> 8) & 0xFF; cvalues[4 * i + 3] = (values[i] >> 0) & 0xFF; } is.read(cvalues, length); DefaultInputBitStream ibs(ios, 16384); cout << endl << endl << "Read:" << endl; bool ok = true; for (int i = 0; i < length; i++) { int x = (int)ibs.readBits((1 + (i % 30))); cout << x; cout << ((x == values[i]) ? " " : "* "); ok &= (x == values[i]); if ((i % 20) == 19) cout << endl; } delete[] cvalues; ibs.close(); res = (ok == true) ? 0 : 1; cout << endl; cout << endl << "Bits written: " << dbs.written() << endl; cout << endl << "Bits read: " << ibs.read() << endl; cout << endl << "\n" << (ok ? "Success" : "Failure") << endl; cout << endl; cout << endl; if (test == 10) { try { cout << "\nTrying to read from closed stream" << endl; ibs.readBit(); } catch (const BitStreamException& e) { cout << "\nException: " << e.what() << endl; } } } delete[] values; return res; } int testBitStreamSpeed1(const string& fileName) { // Test speed cout << "\nSpeed Test1" << endl; int values[] = { 3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9, 3, 31, 14, 41, 15, 59, 92, 26, 65, 53, 35, 58, 89, 97, 79, 93, 32 }; int iter = 150; uint64 written = 0; uint64 read = 0; double delta1 = 0, delta2 = 0; int nn = 100000 * 32; for (int test = 1; test <= iter; test++) { ofstream os(fileName.c_str(), std::ofstream::binary); DefaultOutputBitStream obs(os, 1024 * 1024); clock_t before = clock(); for (int i = 0; i < nn; i++) { obs.writeBits((uint64)values[i % 32], 1 + (i & 63)); } // Close first to force flush() obs.close(); os.close(); clock_t after = clock(); delta1 += (after - before); written += obs.written(); ifstream is(fileName.c_str(), std::ifstream::binary); DefaultInputBitStream ibs(is, 1024 * 1024); before = clock(); for (int i = 0; i < nn; i++) { ibs.readBits(1 + (i & 63)); } ibs.close(); is.close(); after = clock(); delta2 += (after - before); read += ibs.read(); } // MB = 1000 * 1000, MiB = 1024 * 1024 double d = 8.0 * 1024.0 * 1024.0; cout << written << " bits written (" << (written / 1024 / 1024 / 8) << " MB)" << endl; cout << read << " bits read (" << (read / 1024 / 1024 / 8) << " MB)" << endl; cout << endl; cout << "Write [ms] : " << (int)(delta1 / CLOCKS_PER_SEC * 1000) << endl; cout << "Throughput [MiB/s] : " << (int)((double)written / d / (delta1 / CLOCKS_PER_SEC)) << endl; cout << "Read [ms] : " << (int)(delta2 / CLOCKS_PER_SEC * 1000) << endl; cout << "Throughput [MiB/s] : " << (int)((double)read / d / (delta2 / CLOCKS_PER_SEC)) << endl; return 0; } int testBitStreamCorrectnessAligned2() { // Test correctness (kanzi::byte aligned) cout << "Correctness Test - write array - byte aligned" << endl; const int length = 100; kanzi::byte* input = new kanzi::byte[length]; kanzi::byte* output = new kanzi::byte[length]; int res = 0; srand((uint)time(nullptr)); cout << "\nInitial" << endl; for (int test = 1; test <= 10; test++) { stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios, 16384); DebugOutputBitStream dbs(obs, cout); dbs.showByte(true); for (int i = 0; i < length; i++) { input[i] = (kanzi::byte) rand(); cout << (int(input[i]) & 0xFF) << " "; if ((i % 20) == 19) cout << endl; } cout << endl << endl; uint count = 8 + test*(20+(test&1)) + (test&3); dbs.writeBits(input, count); cout << obs.written() << endl; // Close first to force flush() dbs.close(); if (test == 10) { try { cout << "\nTrying to write to closed stream" << endl; dbs.writeBit(1); } catch (const BitStreamException& e) { cout << "\nException: " << e.what() << endl; } } ios.rdbuf()->pubseekpos(0); istringstream is; char* cvalues = new char[length]; for (int i = 0; i < length; i++) { cvalues[i] = char(input[i]) & 0xFF; } is.read(cvalues, length); DefaultInputBitStream ibs(ios, 16384); cout << endl << endl << "Read:" << endl; uint r = ibs.readBits(output, count); bool ok = r == count; if (ok == true) { for (uint i = 0; i < (r>>3); i++) { cout << (int(output[i]) & 0xFF); cout << ((output[i] == input[i]) ? " " : "* "); ok &= (output[i] == input[i]); if ((i % 20) == 19) cout << endl; } } delete[] cvalues; ibs.close(); res = (ok == true) ? 0 : 1; cout << endl; cout << endl << "Bits written: " << dbs.written() << endl; cout << endl << "Bits read: " << ibs.read() << endl; cout << endl << "\n" << (ok ? "Success" : "Failure") << endl; cout << endl; cout << endl; if (test == 10) { try { cout << "\nTrying to read from closed stream" << endl; ibs.readBit(); } catch (const BitStreamException& e) { cout << "\nException: " << e.what() << endl; } } } delete[] input; delete[] output; return res; } int testBitStreamCorrectnessMisaligned2() { // Test correctness (not kanzi::byte aligned) cout << "Correctness Test - write array - not byte aligned" << endl; const int length = 100; kanzi::byte* input = new kanzi::byte[length]; kanzi::byte* output = new kanzi::byte[length]; int res = 0; srand((uint)time(nullptr)); cout << "\nInitial" << endl; for (int test = 1; test <= 10; test++) { stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios, 16384); DebugOutputBitStream dbs(obs, cout); dbs.showByte(true); for (int i = 0; i < length; i++) { input[i] = (kanzi::byte) rand(); cout << (int(input[i]) & 0xFF) << " "; if ((i % 20) == 19) cout << endl; } cout << endl << endl; uint count = 8 + test*(20+(test&1)) + (test&3); dbs.writeBit(0); dbs.writeBits(&input[1], count); // Close first to force flush() dbs.close(); ios.rdbuf()->pubseekpos(0); istringstream is; char* cvalues = new char[4 * length]; for (int i = 0; i < length; i++) { cvalues[i] = char(input[i]) & 0xFF; } is.read(cvalues, length); DefaultInputBitStream ibs(ios, 16384); cout << endl << endl << "Read:" << endl; ibs.readBit(); uint r = ibs.readBits(&output[1], count); bool ok = r == count; if (ok == true) { for (uint i = 1; i < 1 + (r>>3); i++) { cout << (int(output[i]) & 0xFF); cout << ((output[i] == input[i]) ? " " : "* "); ok &= (output[i] == input[i]); if ((i % 20) == 19) cout << endl; } } delete[] cvalues; ibs.close(); res = (ok == true) ? 0 : 1; cout << endl; cout << endl << "Bits written: " << dbs.written() << endl; cout << endl << "Bits read: " << ibs.read() << endl; cout << endl << "\n" << (ok ? "Success" : "Failure") << endl; cout << endl; cout << endl; } delete[] input; delete[] output; return res; } int testSeek(const string& name) { #if !defined(_MSC_VER) || _MSC_VER > 1500 // Test correctness (not kanzi::byte aligned) cout << endl << "Seek Test" << endl << endl; kanzi::byte input[256]; kanzi::byte output[256]; for (int i = 0; i < 256; i++) input[i] = kanzi::byte(i); cout << "Test OutputBitStream" << endl; ofstream ofs(name.c_str(), ios_base::out | ios_base::binary); DefaultOutputBitStream obs(ofs); for (int i = 0; i < 128; i++) obs.writeBits(uint64(0xAA), 8); obs.seek(8 * 32); obs.writeBits(&input[10], 8 * 32); obs.seek(8 * 2); obs.writeBits(&input[100], 8 * 32); obs.close(); ofs.close(); cout << "Bits written: " << obs.written() << endl; ifstream chk(name.c_str(), ios_base::in | ios_base::binary); chk.read(reinterpret_cast(output), 128); chk.close(); for (int i = 0; i < 128; i++) { kanzi::byte expected = kanzi::byte(0xAA); if ((i >= 2) && (i < 34)) expected = input[98 + i]; else if ((i >= 32) && (i < 64)) expected = input[i - 22]; if (output[i] != expected) { cout << "Write failure at index " << i << endl; remove(name.c_str()); return 1; } } remove(name.c_str()); cout << endl; cout << "Test InputBitStream" << endl; for (int i = 0; i < 256; i++) input[i] = kanzi::byte(i); ofstream ofs2(name.c_str(), ios_base::out | ios_base::binary); ofs2.write(reinterpret_cast(input), 256); ofs2.close(); ifstream ifs(name.c_str(), ios_base::in | ios_base::binary); DefaultInputBitStream ibs(ifs); memset(output, 0, 256); ibs.readBits(&output[0], 8 * 16); for (int i = 0; i < 16; i++) { if (output[i] != kanzi::byte(i)) { cout << "Read failure" << endl; remove(name.c_str()); return 1; } } // Positions in bytes int64 positions[5] = { 50, 0, 20, 33, 0 }; memset(output, 0, 256); for (int i = 0; i < 5; i ++) { int64 pos = positions[i]; cout << "Seek " << pos << endl; ibs.seek(8 * pos); if (ibs.tell() != 8 * pos) { cout << "Seek/tell mismatch" << endl; remove(name.c_str()); return 2; } if (ibs.tell() != 8 * ifs.tellg()) { cout << "Seek/tell mismatch" << endl; remove(name.c_str()); return 3; } cout << "Read bits at position " << pos << endl; ibs.readBits(&output[pos], 8 * 10); int64 r = ibs.readBits(8); if (r != pos + 10) { cout << "Incorrect number of read bits" << endl; remove(name.c_str()); return 4; } for (int j = 0; j < 10; j++) { if (output[pos + j] != kanzi::byte(pos + j)) { cout << "Read failure" << endl; remove(name.c_str()); return 5; } } cout << "OK" << endl; } cout << "Bits read: " << ibs.read() << endl; remove(name.c_str()); cout << endl << "Success" << endl; #endif return 0; } int testBitStreamSpeed2(const string& fileName) { // Test speed cout << "\nSpeed Test2" << endl; kanzi::byte values[] = { (kanzi::byte)3, (kanzi::byte)1, (kanzi::byte)4, (kanzi::byte)1, (kanzi::byte)5,(kanzi::byte) 9, (kanzi::byte)2, (kanzi::byte)6, (kanzi::byte)5, (kanzi::byte)3, (kanzi::byte)5, (kanzi::byte)8, (kanzi::byte)9, (kanzi::byte)7, (kanzi::byte)9, (kanzi::byte)3, (kanzi::byte)31, (kanzi::byte)14, (kanzi::byte)41, (kanzi::byte)15, (kanzi::byte)59, (kanzi::byte)92, (kanzi::byte)26, (kanzi::byte)65, (kanzi::byte)53, (kanzi::byte)35, (kanzi::byte)58, (kanzi::byte)89, (kanzi::byte)97, (kanzi::byte)79, (kanzi::byte)93, (kanzi::byte)32 }; int iter = 150; uint64 written = 0; uint64 read = 0; double delta1 = 0, delta2 = 0; kanzi::byte* input = new kanzi::byte[3250000*32]; kanzi::byte* output = new kanzi::byte[3250000*32]; for (int i = 0; i < 3250000; i++) { memcpy(&input[i*32], &values[0], 32); } for (int test = 1; test <= iter; test++) { ofstream os(fileName.c_str(), std::ofstream::binary); DefaultOutputBitStream obs(os, 1024 * 1024); clock_t before = clock(); obs.writeBits(input, 3250000*32); // Close first to force flush() obs.close(); os.close(); clock_t after = clock(); delta1 += (after - before); written += obs.written(); ifstream is(fileName.c_str(), std::ifstream::binary); DefaultInputBitStream ibs(is, 1024 * 1024); before = clock(); ibs.readBits(output, 3250000*32); ibs.close(); is.close(); after = clock(); delta2 += (after - before); read += ibs.read(); } // MiB = 1024 * 1024, MB = 1000 * 1000 double d = 1024.0 * 8192.0; cout << written << " bits written (" << (written / 1024 / 1024 / 8) << " MiB)" << endl; cout << read << " bits read (" << (read / 1024 / 1024 / 8) << " MiB)" << endl; cout << endl; cout << "Write [ms] : " << (int)(delta1 / CLOCKS_PER_SEC * 1000) << endl; cout << "Throughput [MiB/s] : " << (int)((double)written / d / (delta1 / CLOCKS_PER_SEC)) << endl; cout << "Read [ms] : " << (int)(delta2 / CLOCKS_PER_SEC * 1000) << endl; cout << "Throughput [MiB/s] : " << (int)((double)read / d / (delta2 / CLOCKS_PER_SEC)) << endl; delete[] input; delete[] output; return 0; } int testHasMoreToRead() { cout << endl << "hasMoreToRead Test" << endl << endl; stringbuf buffer; iostream ios(&buffer); const char value = char(0xAB); ios.write(&value, 1); ios.rdbuf()->pubseekpos(0); DefaultInputBitStream ibs(ios, 1024); if (ibs.hasMoreToRead() == false) { cout << "Unexpected end of stream" << endl; return 1; } if (ibs.hasMoreToRead() == false) { cout << "Repeated probe lost buffered data" << endl; return 2; } if (ibs.readBits(8) != 0xAB) { cout << "Read failure" << endl; return 3; } if (ibs.hasMoreToRead() == true) { cout << "Unexpected data after end of stream" << endl; return 4; } cout << "Success" << endl; return 0; } int testInputErrorState() { cout << endl << "Input Error-State Test" << endl << endl; stringbuf buffer; iostream ios(&buffer); ios.setstate(ios_base::failbit); DefaultInputBitStream ibs(ios, 1024); try { ibs.hasMoreToRead(); cout << "Expected input error" << endl; return 1; } catch (const BitStreamException& e) { if (e.error() != BitStreamException::INPUT_OUTPUT) { cout << "Unexpected exception code: " << e.error() << endl; return 2; } } cout << "Success" << endl; return 0; } int testOutputErrorState() { cout << endl << "Output Error-State Test" << endl << endl; stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios, 1024); obs.writeBits(0xAB, 8); ios.setstate(ios_base::failbit); try { obs.close(); cout << "Expected output error" << endl; return 1; } catch (const BitStreamException& e) { if (e.error() != BitStreamException::INPUT_OUTPUT) { cout << "Unexpected exception code: " << e.error() << endl; return 2; } } cout << "Success" << endl; return 0; } #ifdef __GNUG__ int main(int argc, const char* argv[]) #else int TestDefaultBitStream_main(int argc, const char* argv[]) #endif { bool doPerf = true; if (argc <= 1) { cout << "Missing temp output file" << endl; exit(1); } if (argc > 2) { string str = argv[2]; transform(str.begin(), str.end(), str.begin(), safeToUpper); doPerf = str != "-NOPERF"; } int res = 0; try { string fileName = argv[1]; struct STAT buffer; if ((STAT(fileName.c_str(), &buffer) == 0) && S_ISDIR(buffer.st_mode)) { cout << "Temp output path must be a file, not a directory" << endl; return 1; } res |= testBitStreamCorrectnessAligned1(); res |= testBitStreamCorrectnessAligned2(); res |= testBitStreamCorrectnessMisaligned1(); res |= testBitStreamCorrectnessMisaligned2(); res |= testHasMoreToRead(); res |= testInputErrorState(); res |= testOutputErrorState(); res |= testSeek(fileName); if (doPerf == true) { res |= testBitStreamSpeed1(fileName); res |= testBitStreamSpeed2(fileName); } } catch (const kanzi::IOException& e) { cout << "Exception: " << e.what() << endl; res = 99; } catch (const BitStreamException& e) { cout << "Exception: " << e.what() << endl; res = 99; } return res; } kanzi-cpp-2.6.0/src/test/TestEntropyCodec.cpp000066400000000000000000000643661525576310500211530ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include "../types.hpp" #include "../util/strings.hpp" #include "../entropy/HuffmanEncoder.hpp" #include "../entropy/RangeEncoder.hpp" #include "../entropy/ANSRangeEncoder.hpp" #include "../entropy/BinaryEntropyEncoder.hpp" #include "../entropy/ExpGolombEncoder.hpp" #include "../entropy/FPAQEncoder.hpp" #include "../entropy/EntropyUtils.hpp" #include "../bitstream/DefaultOutputBitStream.hpp" #include "../bitstream/DefaultInputBitStream.hpp" #include "../bitstream/DebugOutputBitStream.hpp" #include "../entropy/HuffmanDecoder.hpp" #include "../entropy/RangeDecoder.hpp" #include "../entropy/ANSRangeDecoder.hpp" #include "../entropy/BinaryEntropyDecoder.hpp" #include "../entropy/ExpGolombDecoder.hpp" #include "../entropy/FPAQDecoder.hpp" #include "../entropy/CMPredictor.hpp" #include "../entropy/TPAQPredictor.hpp" using namespace kanzi; using namespace std; static EntropyEncoder* getEncoder(string name, OutputBitStream& obs, Predictor* predictor); static EntropyDecoder* getDecoder(string name, InputBitStream& ibs, Predictor* predictor); static void copyBits(InputBitStream& ibs, OutputBitStream& obs, uint64 count) { while (count >= 64) { obs.writeBits(ibs.readBits(64), 64); count -= 64; } if (count > 0) obs.writeBits(ibs.readBits(uint(count)), uint(count)); } static string encodeEntropyPayload(const string& name, const kanzi::byte block[], uint size) { stringbuf encoded; iostream ios(&encoded); DefaultOutputBitStream obs(ios, 16384); EntropyEncoder* ec = getEncoder(name, obs, NULL); if (ec == NULL) return ""; const int res = ec->encode(block, 0, size); ec->dispose(); delete ec; obs.close(); return (res == int(size)) ? encoded.str() : ""; } static string shrinkFPAQDeclaredSize(const string& data) { istringstream is(data); DefaultInputBitStream ibs(is, 16384); stringbuf mutated; iostream ios(&mutated); DefaultOutputBitStream obs(ios, 16384); const uint32 sz = EntropyUtils::readVarInt(ibs); if (sz == 0) return ""; EntropyUtils::writeVarInt(obs, sz - 1); copyBits(ibs, obs, uint64(data.size()) * 8 - ibs.read()); obs.close(); return mutated.str(); } static string zeroFPAQDeclaredSize(const string& data) { istringstream is(data); DefaultInputBitStream ibs(is, 16384); stringbuf mutated; iostream ios(&mutated); DefaultOutputBitStream obs(ios, 16384); const uint32 sz = EntropyUtils::readVarInt(ibs); if (sz == 0) return ""; EntropyUtils::writeVarInt(obs, 0); copyBits(ibs, obs, uint64(data.size()) * 8 - ibs.read()); obs.close(); return mutated.str(); } static string shrinkANSDeclaredSize(const string& data) { istringstream is(data); DefaultInputBitStream ibs(is, 16384); stringbuf mutated; iostream ios(&mutated); DefaultOutputBitStream obs(ios, 16384); const uint logRange = uint(ibs.readBits(3)); obs.writeBits(logRange, 3); uint alphabet[256]; const int alphabetSize = EntropyUtils::decodeAlphabet(ibs, alphabet); if (EntropyUtils::encodeAlphabet(obs, alphabet, 256, alphabetSize) < 0) return ""; if (alphabetSize > 1) { const int chkSize = (alphabetSize >= 64) ? 8 : 6; for (int i = 1; i < alphabetSize; i += chkSize) { const uint logMax = uint(ibs.readBits(4)); obs.writeBits(logMax, 4); if (logMax == 0) continue; const int endj = min(i + chkSize, alphabetSize); for (int j = i; j < endj; j++) obs.writeBits(ibs.readBits(logMax), logMax); } } const uint32 sz = EntropyUtils::readVarInt(ibs); if (sz == 0) return ""; EntropyUtils::writeVarInt(obs, sz - 1); copyBits(ibs, obs, uint64(data.size()) * 8 - ibs.read()); obs.close(); return mutated.str(); } static string shrinkHuffmanDeclaredSize(const string& data) { istringstream is(data); DefaultInputBitStream ibs(is, 16384); stringbuf mutated; iostream ios(&mutated); DefaultOutputBitStream obs(ios, 16384); uint alphabet[256]; const int alphabetSize = EntropyUtils::decodeAlphabet(ibs, alphabet); if (EntropyUtils::encodeAlphabet(obs, alphabet, 256, alphabetSize) < 0) return ""; if (alphabetSize <= 1) return ""; ExpGolombDecoder egdec(ibs, true); ExpGolombEncoder egenc(obs, true); for (int i = 0; i < alphabetSize; i++) { const kanzi::byte delta = egdec.decodeByte(); egenc.encodeByte(delta); } uint32 sz[4]; int idx = -1; for (int i = 0; i < 4; i++) { sz[i] = EntropyUtils::readVarInt(ibs); if ((sz[i] > 0) && ((idx < 0) || (sz[i] > sz[idx]))) idx = i; } if (idx < 0) return ""; for (int i = 0; i < 4; i++) EntropyUtils::writeVarInt(obs, sz[i] - ((i == idx) ? 1 : 0)); copyBits(ibs, obs, uint64(data.size()) * 8 - ibs.read()); obs.close(); return mutated.str(); } static bool malformedEntropyIsRejected(const string& name, const string& data, uint size) { istringstream is(data); DefaultInputBitStream ibs(is, 16384); EntropyDecoder* ed = getDecoder(name, ibs, NULL); if (ed == NULL) return false; vector decoded(size, kanzi::byte(0)); try { const int res = ed->decode(&decoded[0], 0, size); ed->dispose(); delete ed; ibs.close(); return res != int(size); } catch (const exception&) { delete ed; try { ibs.close(); } catch (const exception&) { } return true; } } int testDeclaredPayloadConsumption() { cout << endl << "=== Declared payload consumption test ===" << endl; const uint size = 4096; vector values(size); for (uint i = 0; i < size; i++) values[i] = kanzi::byte(((i * 13) ^ (i >> 3) ^ ((i & 15) << 4)) & 0xFF); struct TestCase { const char* name; string (*mutator)(const string&); }; TestCase tests[3] = { { "HUFFMAN", shrinkHuffmanDeclaredSize }, { "ANS0", shrinkANSDeclaredSize }, { "FPAQ", shrinkFPAQDeclaredSize } }; for (int i = 0; i < 3; i++) { const string encoded = encodeEntropyPayload(tests[i].name, &values[0], size); if (encoded.empty()) { cout << "Could not encode test payload for " << tests[i].name << endl; return 1; } const string mutated = tests[i].mutator(encoded); if (mutated.empty()) { cout << "Could not mutate test payload for " << tests[i].name << endl; return 2; } if (malformedEntropyIsRejected(tests[i].name, mutated, size) == false) { cout << "Malformed payload accepted for " << tests[i].name << endl; return 3; } } cout << "Declared payload consumption test passed" << endl; return 0; } int testFPAQZeroDeclaredSize() { cout << endl << "=== FPAQ zero declared size test ===" << endl; const uint size = 1 << 20; vector values(size); for (uint i = 0; i < size; i++) values[i] = kanzi::byte((i * 17) & 0xFF); const string encoded = encodeEntropyPayload("FPAQ", &values[0], size); if (encoded.empty()) { cout << "Could not encode test payload for FPAQ" << endl; return 1; } const string mutated = zeroFPAQDeclaredSize(encoded); if (mutated.empty()) { cout << "Could not mutate test payload for FPAQ" << endl; return 2; } if (malformedEntropyIsRejected("FPAQ", mutated, size) == false) { cout << "Malformed zero-sized FPAQ payload accepted" << endl; return 3; } cout << "FPAQ zero declared size test passed" << endl; return 0; } int testANS1MissingContext() { cout << endl << "=== ANS1 implicit context test ===" << endl; const uint size = 40; const uint alphabet[] = { 1 }; // Exercise both freshly allocated tables and tables from an earlier chunk. for (int warm = 0; warm < 2; warm++) { stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios, 16384); vector previous(size, kanzi::byte(1)); if (warm != 0) { ANSRangeEncoder encoder(obs, 1); encoder.encode(&previous[0], 0, size); } // Context 0 emits 1; empty context 1 implicitly emits 0. obs.writeBits(uint64(0), 3); // log range = 8 EntropyUtils::encodeAlphabet(obs, alphabet, 256, 1); for (int i = 1; i < 256; i++) EntropyUtils::encodeAlphabet(obs, alphabet, 256, 0); EntropyUtils::writeVarInt(obs, 8); for (int i = 0; i < 4; i++) obs.writeBits(uint32(1 << 15), 32); // Each lane renormalizes once, on its first symbol. const kanzi::byte payload[8] = { kanzi::byte(0) }; obs.writeBits(payload, 64); obs.close(); istringstream is(buffer.str()); DefaultInputBitStream ibs(is, 16384); ANSRangeDecoder decoder(ibs, 1); vector decoded(size); if (warm != 0) { if ((decoder.decode(&decoded[0], 0, size) != int(size)) || (decoded != previous)) return 1; } if (decoder.decode(&decoded[0], 0, size) != int(size)) return 1; for (uint i = 0; i < size; i++) { if (decoded[i] != kanzi::byte(1 - ((i % (size / 4)) & 1))) { cout << "Incorrect ANS1 implicit context output" << endl; return 1; } } } cout << "ANS1 implicit context test passed" << endl; return 0; } int testHuffmanFragmentedRoundTrip() { cout << endl << "=== Huffman fragmented round-trip test ===" << endl; const uint size = 1649; vector values(size); vector decoded(size, kanzi::byte(0)); uint32 state = 3454687338U; for (uint i = 0; i < size; i++) { state = (state * 1103515245U) + 12345U; const uint32 val0 = state >> 24; state = (state * 1103515245U) + 12345U; const uint32 val1 = state >> 24; values[i] = (val0 < 192) ? kanzi::byte(val1 & 15) : kanzi::byte(val1); } stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios, 1 << 15); HuffmanEncoder encoder(obs); if (encoder.encode(&values[0], 0, size) != int(size)) { cout << "Encoding error in Huffman fragmented round-trip test" << endl; return 1; } encoder.dispose(); obs.close(); ios.rdbuf()->pubseekpos(0); DefaultInputBitStream ibs(ios, 1 << 15); HuffmanDecoder decoder(ibs); if (decoder.decode(&decoded[0], 0, size) != int(size)) { cout << "Decoding error in Huffman fragmented round-trip test" << endl; return 1; } decoder.dispose(); ibs.close(); if (memcmp(&values[0], &decoded[0], size) != 0) { cout << "Mismatch in Huffman fragmented round-trip test" << endl; return 1; } cout << "Huffman fragmented round-trip test passed" << endl; return 0; } class ConstantPredictor FINAL : public Predictor { public: explicit ConstantPredictor(int value) : _value(value) {} void update(int) {} int get() { return _value; } private: int _value; }; static Predictor* getPredictor(string type) { if (type.compare("TPAQ") == 0) return new TPAQPredictor(); if (type.compare("TPAQX") == 0) return new TPAQPredictor(); if (type.compare("CM") == 0) return new CMPredictor(); return nullptr; } static EntropyEncoder* getEncoder(string name, OutputBitStream& obs, Predictor* predictor) { if (name.compare("HUFFMAN") == 0) return new HuffmanEncoder(obs); if (name.compare("ANS0") == 0) return new ANSRangeEncoder(obs, 0); if (name.compare("ANS1") == 0) return new ANSRangeEncoder(obs, 1); if (name.compare("RANGE") == 0) return new RangeEncoder(obs); if (name.compare("EXPGOLOMB") == 0) return new ExpGolombEncoder(obs); if (name.compare("FPAQ") == 0) return new FPAQEncoder(obs); if (predictor != nullptr) { if (name.compare("TPAQ") == 0) return new BinaryEntropyEncoder(obs, predictor, true); if (name.compare("CM") == 0) return new BinaryEntropyEncoder(obs, predictor, true); } cout << "No such entropy encoder: " << name << endl; return nullptr; } static EntropyDecoder* getDecoder(string name, InputBitStream& ibs, Predictor* predictor) { if (name.compare("HUFFMAN") == 0) return new HuffmanDecoder(ibs); if (name.compare("ANS0") == 0) return new ANSRangeDecoder(ibs, 0); if (name.compare("ANS1") == 0) return new ANSRangeDecoder(ibs, 1); if (name.compare("RANGE") == 0) return new RangeDecoder(ibs); if (name.compare("FPAQ") == 0) return new FPAQDecoder(ibs); if (predictor != nullptr) { if (name.compare("TPAQ") == 0) return new BinaryEntropyDecoder(ibs, predictor, true); if (name.compare("CM") == 0) return new BinaryEntropyDecoder(ibs, predictor, true); } if (name.compare("EXPGOLOMB") == 0) return new ExpGolombDecoder(ibs); cout << "No such entropy decoder: " << name << endl; return nullptr; } static int testBinaryPayloadBoundary() { cout << endl << "=== Binary entropy payload boundary test ===" << endl; const uint size = 1 << 16; vector values(size, kanzi::byte(0)); stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios, 16384); BinaryEntropyEncoder encoder(obs, new ConstantPredictor(4095), true); if (encoder.encode(&values[0], 0, size) != int(size)) return 1; encoder.dispose(); obs.close(); const string mutated = zeroFPAQDeclaredSize(buffer.str()); if (mutated.empty()) return 2; istringstream is(mutated); DefaultInputBitStream ibs(is, 16384); BinaryEntropyDecoder decoder(ibs, new ConstantPredictor(4095), true); vector decoded(size, kanzi::byte(0)); bool accepted = false; try { accepted = decoder.decode(&decoded[0], 0, size) == int(size); } catch (const exception&) { // Expected: the first four-byte refill crosses the declared boundary. } decoder.dispose(); ibs.close(); if (accepted) { cout << "Malformed binary entropy payload was accepted" << endl; return 3; } cout << "Binary entropy payload boundary test passed" << endl; return 0; } int testEntropyCodecCorrectness(const string& name) { // Test behavior cout << "=== Correctness test for " << name << " ===" << endl; srand((uint)time(nullptr)); int res = 0; for (int ii = 1; ii < 50; ii++) { cout << endl << endl << "Test " << ii << endl; kanzi::byte val[256]; int size = 40; if (ii == 3) { kanzi::byte val2[] = { (kanzi::byte)0, (kanzi::byte)0, (kanzi::byte)32, (kanzi::byte)15, (kanzi::byte)-4, (kanzi::byte)16, (kanzi::byte)0, (kanzi::byte)16, (kanzi::byte)0, (kanzi::byte)7, (kanzi::byte)-1, (kanzi::byte)-4, (kanzi::byte)-32, (kanzi::byte)0, (kanzi::byte)31, (kanzi::byte)-1 }; size = 16; memcpy(val, &val2[0], size); } else if (ii == 2) { kanzi::byte val2[] = { (kanzi::byte)0x3d, (kanzi::byte)0x4d, (kanzi::byte)0x54, (kanzi::byte)0x47, (kanzi::byte)0x5a, (kanzi::byte)0x36, (kanzi::byte)0x39, (kanzi::byte)0x26, (kanzi::byte)0x72, (kanzi::byte)0x6f, (kanzi::byte)0x6c, (kanzi::byte)0x65, (kanzi::byte)0x3d, (kanzi::byte)0x70, (kanzi::byte)0x72, (kanzi::byte)0x65 }; size = 16; memcpy(val, &val2[0], size); } else if (ii == 1) { for (int i = 0; i < size; i++) val[i] = kanzi::byte(2); // all identical } else if (ii == 4) { for (int i = 0; i < size; i++) val[i] = kanzi::byte(2 + (i & 1)); // 2 symbols } else if (ii == 5) { size = 1; val[0] = kanzi::byte(42); } else if (ii == 6) { size = 2; val[0] = kanzi::byte(42); val[1] = kanzi::byte(42); } else if (ii == 7) { for (int i = 0; i < 44; i++) val[i] = kanzi::byte(i & 7); } else { size = 256; for (int i = 0; i < 256; i++) val[i] = kanzi::byte(64 + 4 * ii + (rand() % (8*ii + 1))); } kanzi::byte* values = &val[0]; cout << "Original:" << endl; for (int i = 0; i < size; i++) cout << int(values[i]) << " "; cout << endl << endl << "Encoded:" << endl; stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios); DebugOutputBitStream dbgobs(obs); dbgobs.showByte(true); EntropyEncoder* ec = getEncoder(name, dbgobs, getPredictor(name)); if (ec == nullptr) return 1; ec->encode(values, 0, size); ec->dispose(); delete ec; dbgobs.close(); ios.rdbuf()->pubseekpos(0); DefaultInputBitStream ibs(ios); EntropyDecoder* ed = getDecoder(name, ibs, getPredictor(name)); if (ed == nullptr) return 1; cout << endl << endl << "Decoded:" << endl; bool ok = true; kanzi::byte* values2 = new kanzi::byte[size]; ed->decode(values2, 0, size); ed->dispose(); delete ed; ibs.close(); for (int j = 0; j < size; j++) { if (values[j] != values2[j]) ok = false; cout << (int)values2[j] << " "; } cout << endl; cout << (ok ? "Identical" : "Different") << endl; delete[] values2; res = ok ? 0 : 2; } return res; } static int testExpGolombUnsignedRoundTrip() { cout << "=== Correctness test for unsigned EXPGOLOMB ===" << endl; const uint size = 256; kanzi::byte values[size]; for (uint i = 0; i < size; i++) values[i] = kanzi::byte(i); stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios); ExpGolombEncoder encoder(obs, false); if (encoder.encode(values, 0, size) != int(size)) { cout << "Unsigned EXPGOLOMB encoding failed" << endl; return 1; } encoder.dispose(); obs.close(); ios.rdbuf()->pubseekpos(0); DefaultInputBitStream ibs(ios); ExpGolombDecoder decoder(ibs, false); kanzi::byte decoded[size]; if (decoder.decode(decoded, 0, size) != int(size)) { cout << "Unsigned EXPGOLOMB decoding failed" << endl; return 1; } decoder.dispose(); ibs.close(); for (uint i = 0; i < size; i++) { if (values[i] != decoded[i]) { cout << "Unsigned EXPGOLOMB mismatch at index " << i << endl; return 1; } } return 0; } int testEntropyCodecSpeed(const string& name) { // Test speed cout << endl << endl << "=== Speed test for " << name << " ===" << endl; int repeats[] = { 3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5, 8, 9, 7, 9, 3 }; int size = 500000; int iter = 100; int res = 0; srand((uint)time(nullptr)); kanzi::byte values1[500000]; kanzi::byte values2[500000]; for (int jj = 0; jj < 3; jj++) { cout << endl << "Test " << (jj + 1) << endl; double delta1 = 0, delta2 = 0; for (int ii = 0; ii < iter; ii++) { int idx = 0; memset(values1, 0x00, size); memset(values2, 0xAA, size); int n = 0; while (n < size) { int n0 = n; int len = max(min(repeats[idx], size - n), 1); idx = (idx + 1) & 0x0F; kanzi::byte b = (kanzi::byte)(rand() % 255); for (int j = n0; j < n0 + len; j++) { values1[j] = b; n++; } } // Encode stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios, 16384); EntropyEncoder* ec = getEncoder(name, obs, getPredictor(name)); if (ec == nullptr) return 1; clock_t before1 = clock(); if (ec->encode(values1, 0, size) < 0) { cout << "Encoding error" << endl; delete ec; return 1; } ec->dispose(); clock_t after1 = clock(); delta1 += (after1 - before1); delete ec; obs.close(); // Decode ios.rdbuf()->pubseekpos(0); DefaultInputBitStream ibs(ios, 16384); EntropyDecoder* ed = getDecoder(name, ibs, getPredictor(name)); if (ed == nullptr) return 1; clock_t before2 = clock(); if (ed->decode(values2, 0, size) < 0) { cout << "Decoding error" << endl; delete ed; return 1; } ed->dispose(); clock_t after2 = clock(); delta2 += (after2 - before2); delete ed; ibs.close(); // Sanity check for (int i = 0; i < size; i++) { if (values1[i] != values2[i]) { cout << "Error at index " << i << " (" << (int)values1[i] << "<->" << (int)values2[i] << ")" << endl; res = 1; break; } } } // KB = 1000, KiB = 1024 double prod = double(iter) * double(size); double b2KiB = double(1) / double(1024); double d1_sec = delta1 / CLOCKS_PER_SEC; double d2_sec = delta2 / CLOCKS_PER_SEC; cout << "Encode [ms] : " << (int)(d1_sec * 1000) << endl; cout << "Throughput [KiB/s] : " << (int)(prod * b2KiB / d1_sec) << endl; cout << "Decode [ms] : " << (int)(d2_sec * 1000) << endl; cout << "Throughput [KiB/s] : " << (int)(prod * b2KiB / d2_sec) << endl; } return res; } int testBinaryEntropyBufferGrowth() { cout << endl << "=== Buffer growth test for binary entropy codec ===" << endl; const int size = 1 << 20; vector values(size, kanzi::byte(0)); vector decoded(size, kanzi::byte(0xAA)); stringbuf buffer; iostream ios(&buffer); DefaultOutputBitStream obs(ios, 1 << 15); BinaryEntropyEncoder encoder(obs, new ConstantPredictor(4095), true); if (encoder.encode(&values[0], 0, uint(values.size())) != size) { cout << "Encoding error in buffer growth test" << endl; return 1; } encoder.dispose(); obs.close(); ios.rdbuf()->pubseekpos(0); DefaultInputBitStream ibs(ios, 1 << 15); BinaryEntropyDecoder decoder(ibs, new ConstantPredictor(4095), true); if (decoder.decode(&decoded[0], 0, uint(decoded.size())) != size) { cout << "Decoding error in buffer growth test" << endl; return 1; } decoder.dispose(); ibs.close(); if (memcmp(&values[0], &decoded[0], values.size()) != 0) { cout << "Mismatch in buffer growth test" << endl; return 1; } cout << "Buffer growth test passed" << endl; return 0; } #ifdef __GNUG__ int main(int argc, const char* argv[]) #else int TestEntropyCodec_main(int argc, const char* argv[]) #endif { int res = 0; try { res |= testExpGolombUnsignedRoundTrip(); res |= testBinaryEntropyBufferGrowth(); res |= testBinaryPayloadBoundary(); res |= testDeclaredPayloadConsumption(); res |= testFPAQZeroDeclaredSize(); res |= testANS1MissingContext(); res |= testHuffmanFragmentedRoundTrip(); vector codecs; bool doPerf = true; if (argc == 1) { #if __cplusplus < 201103L string allCodecs[] = { "HUFFMAN", "ANS0", "ANS1", "RANGE", "EXPGOLOMB", "CM", "TPAQ" }; const int count = int(sizeof(allCodecs) / sizeof(allCodecs[0])); for (int i = 0; i < count; i++) codecs.push_back(allCodecs[i]); #else codecs = { "HUFFMAN", "ANS0", "ANS1", "RANGE", "EXPGOLOMB", "CM", "TPAQ" }; #endif } else { string str = argv[1]; transform(str.begin(), str.end(), str.begin(), safeToUpper); if (str == "-TYPE=ALL") { #if __cplusplus < 201103L string allCodecs[] = { "HUFFMAN", "ANS0", "ANS1", "RANGE", "EXPGOLOMB", "CM", "TPAQ" }; const int count = int(sizeof(allCodecs) / sizeof(allCodecs[0])); for (int i = 0; i < count; i++) codecs.push_back(allCodecs[i]); #else codecs = { "HUFFMAN", "ANS0", "ANS1", "RANGE", "EXPGOLOMB", "CM", "TPAQ" }; #endif } else { codecs.push_back(str.substr(6)); } if (argc > 2) { str = argv[2]; transform(str.begin(), str.end(), str.begin(), safeToUpper); doPerf = str != "-NOPERF"; } } for (vector::iterator it = codecs.begin(); it != codecs.end(); ++it) { cout << endl << endl << "Test" << *it << endl; res |= testEntropyCodecCorrectness(*it); if (doPerf == true) res |= testEntropyCodecSpeed(*it); } } catch (const exception& e) { cout << e.what() << endl; res = 123; } cout << endl; cout << ((res == 0) ? "Success" : "Failure") << endl; return res; } kanzi-cpp-2.6.0/src/test/TestFactories.cpp000066400000000000000000000215111525576310500204550ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include #include "../Context.hpp" #include "../bitstream/DefaultInputBitStream.hpp" #include "../bitstream/DefaultOutputBitStream.hpp" #include "../entropy/EntropyDecoderFactory.hpp" #include "../entropy/EntropyEncoderFactory.hpp" #include "../transform/TransformFactory.hpp" using namespace std; using namespace kanzi; #define ASSERT_TRUE(cond, msg) \ do { \ if (!(cond)) { \ cerr << "ASSERT FAILED: " << msg << " (" << __FILE__ \ << ":" << __LINE__ << ")" << endl; \ return 1; \ } \ } while (0) static bool expectInvalidArgument(void (*fn)()) { try { fn(); } catch (const invalid_argument&) { return true; } return false; } static void callUnknownTransformType() { TransformFactory::getType("not-a-transform"); } static void callTooManyTransforms() { TransformFactory::getType("LZ+RLT+TEXT+UTF+EXE+PACK+DNA+MM+SRT"); } static void callUnknownEncoderType() { EntropyEncoderFactory::getType("bad-entropy"); } static void callUnknownDecoderType() { EntropyDecoderFactory::getType("bad-entropy"); } static void callUnknownEncoderName() { EntropyEncoderFactory::getName(42); } static void callUnknownDecoderName() { EntropyDecoderFactory::getName(42); } static int testTransformFactory() { cout << "Test TransformFactory" << endl; ASSERT_TRUE(TransformFactory::getTypeToken("text") == TransformFactory::DICT_TYPE, "TEXT transform must be case insensitive"); ASSERT_TRUE(TransformFactory::getType("NONE") == 0, "NONE transform must encode to zero"); ASSERT_TRUE(TransformFactory::getType("NONE+NONE") == 0, "Null transforms must be skipped"); ASSERT_TRUE(TransformFactory::getTransformCount( TransformFactory::getType("LZ")) == 1, "Single transform sequence must contain one transform"); ASSERT_TRUE(TransformFactory::getTransformCount( TransformFactory::getType("LZ+RLT+TEXT+UTF+EXE")) == 5, "Transform count must include all non-null transforms"); ASSERT_TRUE(TransformFactory::getName( TransformFactory::getType("LZ+NONE+RLT")) == "LZ+RLT", "Transform names must round-trip without null transforms"); ASSERT_TRUE(expectInvalidArgument(callUnknownTransformType), "Unknown transform type must throw"); ASSERT_TRUE(expectInvalidArgument(callTooManyTransforms), "More than eight transforms must throw"); { Context ctx; ctx.putString("entropy", "NONE"); TransformSequence* seq = TransformFactory::newTransform(ctx, TransformFactory::getType("TEXT")); ASSERT_TRUE(seq->getNbTransforms() == 1, "TEXT sequence must contain one transform"); ASSERT_TRUE(ctx.getInt("textcodec", 0) == 2, "TEXT must select codec 2 with NONE entropy"); delete seq; } { Context ctx; ctx.putString("entropy", "FPAQ"); TransformSequence* seq = TransformFactory::newTransform(ctx, TransformFactory::getType("TEXT")); ASSERT_TRUE(ctx.getInt("textcodec", 0) == 1, "TEXT must select codec 1 with FPAQ entropy"); delete seq; } { Context ctx; TransformSequence* seq = TransformFactory::newTransform(ctx, TransformFactory::getType("LZX")); ASSERT_TRUE(ctx.getInt("lz", 0) == TransformFactory::LZX_TYPE, "LZX transform must set lz context"); delete seq; } { Context ctx; TransformSequence* seq = TransformFactory::newTransform(ctx, TransformFactory::getType("LZP")); ASSERT_TRUE(ctx.getInt("lz", 0) == TransformFactory::LZP_TYPE, "LZP transform must set lz context"); delete seq; } { Context ctx; TransformSequence* seq = TransformFactory::newTransform(ctx, TransformFactory::getType("DNA")); ASSERT_TRUE(ctx.getInt("packOnlyDNA", 0) == 1, "DNA transform must set packOnlyDNA context"); delete seq; } return 0; } static int testEntropyFactories() { cout << "Test Entropy factories" << endl; ASSERT_TRUE(EntropyEncoderFactory::getType("none") == EntropyEncoderFactory::NONE_TYPE, "NONE entropy encoder must be case insensitive"); ASSERT_TRUE(EntropyDecoderFactory::getType("ans0") == EntropyDecoderFactory::ANS0_TYPE, "ANS0 entropy decoder must be case insensitive"); ASSERT_TRUE(string(EntropyEncoderFactory::getName(EntropyEncoderFactory::HUFFMAN_TYPE)) == "HUFFMAN", "Encoder name must round-trip"); ASSERT_TRUE(string(EntropyDecoderFactory::getName(EntropyDecoderFactory::TPAQX_TYPE)) == "TPAQX", "Decoder name must round-trip"); ASSERT_TRUE(expectInvalidArgument(callUnknownEncoderType), "Unknown encoder type must throw"); ASSERT_TRUE(expectInvalidArgument(callUnknownDecoderType), "Unknown decoder type must throw"); ASSERT_TRUE(expectInvalidArgument(callUnknownEncoderName), "Unknown encoder name must throw"); ASSERT_TRUE(expectInvalidArgument(callUnknownDecoderName), "Unknown decoder name must throw"); stringbuf buffer; iostream io(&buffer); DefaultOutputBitStream obs(io, 16384); Context ctx; EntropyEncoder* encoders[] = { EntropyEncoderFactory::newEncoder(obs, ctx, EntropyEncoderFactory::NONE_TYPE), EntropyEncoderFactory::newEncoder(obs, ctx, EntropyEncoderFactory::HUFFMAN_TYPE), EntropyEncoderFactory::newEncoder(obs, ctx, EntropyEncoderFactory::RANGE_TYPE), EntropyEncoderFactory::newEncoder(obs, ctx, EntropyEncoderFactory::ANS0_TYPE), EntropyEncoderFactory::newEncoder(obs, ctx, EntropyEncoderFactory::ANS1_TYPE), EntropyEncoderFactory::newEncoder(obs, ctx, EntropyEncoderFactory::FPAQ_TYPE), EntropyEncoderFactory::newEncoder(obs, ctx, EntropyEncoderFactory::CM_TYPE), EntropyEncoderFactory::newEncoder(obs, ctx, EntropyEncoderFactory::TPAQ_TYPE), EntropyEncoderFactory::newEncoder(obs, ctx, EntropyEncoderFactory::TPAQX_TYPE) }; for (size_t i = 0; i < sizeof(encoders) / sizeof(encoders[0]); i++) { ASSERT_TRUE(encoders[i] != nullptr, "Entropy encoder must be created"); encoders[i]->dispose(); delete encoders[i]; } obs.close(); io.rdbuf()->pubseekpos(0); DefaultInputBitStream ibs(io, 16384); EntropyDecoder* decoders[] = { EntropyDecoderFactory::newDecoder(ibs, ctx, EntropyDecoderFactory::NONE_TYPE), EntropyDecoderFactory::newDecoder(ibs, ctx, EntropyDecoderFactory::HUFFMAN_TYPE), EntropyDecoderFactory::newDecoder(ibs, ctx, EntropyDecoderFactory::RANGE_TYPE), EntropyDecoderFactory::newDecoder(ibs, ctx, EntropyDecoderFactory::ANS0_TYPE), EntropyDecoderFactory::newDecoder(ibs, ctx, EntropyDecoderFactory::ANS1_TYPE), EntropyDecoderFactory::newDecoder(ibs, ctx, EntropyDecoderFactory::FPAQ_TYPE), EntropyDecoderFactory::newDecoder(ibs, ctx, EntropyDecoderFactory::CM_TYPE), EntropyDecoderFactory::newDecoder(ibs, ctx, EntropyDecoderFactory::TPAQ_TYPE), EntropyDecoderFactory::newDecoder(ibs, ctx, EntropyDecoderFactory::TPAQX_TYPE) }; for (size_t i = 0; i < sizeof(decoders) / sizeof(decoders[0]); i++) { ASSERT_TRUE(decoders[i] != nullptr, "Entropy decoder must be created"); decoders[i]->dispose(); delete decoders[i]; } ibs.close(); return 0; } int main() { if (testTransformFactory() != 0) return 1; if (testEntropyFactories() != 0) return 1; cout << "All factory tests passed." << endl; return 0; } kanzi-cpp-2.6.0/src/test/TestMalformedStream.cpp000066400000000000000000000207221525576310500216230ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include "../Error.hpp" #include "../bitstream/DefaultOutputBitStream.hpp" #include "../entropy/EntropyDecoderFactory.hpp" #include "../io/CompressedInputStream.hpp" #include "../io/IOException.hpp" #include "../transform/TransformFactory.hpp" using namespace std; using namespace kanzi; #define ASSERT_TRUE(cond, msg) \ do { \ if (!(cond)) { \ cerr << "ASSERT FAILED: " << msg << " (" << __FILE__ \ << ":" << __LINE__ << ")" << endl; \ return 1; \ } \ } while (0) static uint32 computeHeaderChecksum(int bsVersion, uint64 checksumSize, short entropyType, uint64 transformType, int blockSize, int szMask, uint64 outputSize) { uint32 seed = uint32((bsVersion >= 6 ? 0x01030507 : 1) * bsVersion); const uint32 hash = 0x1E35A7BD; uint32 checksum = hash * seed; if (bsVersion >= 6) checksum ^= hash * uint32(~checksumSize); checksum ^= hash * uint32(~entropyType); checksum ^= hash * uint32((~transformType) >> 32); checksum ^= hash * uint32(~transformType); checksum ^= hash * uint32(~blockSize); if (szMask != 0) { checksum ^= hash * uint32((~outputSize) >> 32); checksum ^= hash * uint32(~outputSize); } return (checksum >> 23) ^ (checksum >> 3); } static string buildHeader(int type, int bsVersion, uint64 checksumSize, short entropyType, uint64 transformType, int blockSize, int szMask, uint64 outputSize, bool validChecksum) { stringbuf buffer; iostream io(&buffer); DefaultOutputBitStream obs(io, 16384); const int crcSize = bsVersion <= 5 ? 16 : 24; obs.writeBits(uint32(type), 32); obs.writeBits(uint32(bsVersion), 4); if (bsVersion >= 6) obs.writeBits(checksumSize, 2); else obs.writeBit(checksumSize == 0 ? 0 : 1); obs.writeBits(uint64(entropyType), 5); obs.writeBits(transformType, 48); obs.writeBits(uint64(blockSize >> 4), 28); obs.writeBits(uint64(szMask), 2); if (szMask != 0) obs.writeBits(outputSize, 16 * szMask); if (bsVersion >= 6) obs.writeBits(uint64(0), 15); uint32 checksum = computeHeaderChecksum(bsVersion, checksumSize, entropyType, transformType, blockSize, szMask, outputSize); if (validChecksum == false) checksum ^= 1; obs.writeBits(checksum, crcSize); obs.close(); return buffer.str(); } static string buildMalformedBlockStream() { const int version = 6; const int type = 0x4B414E5A; const int blockSize = 1024; const short entropy = EntropyDecoderFactory::NONE_TYPE; const uint64 transform = uint64(TransformFactory::NONE_TYPE) << 42; const string header = buildHeader(type, version, 0, entropy, transform, blockSize, 0, 0, true); stringbuf blockBuffer; iostream blockIO(&blockBuffer); DefaultOutputBitStream blockObs(blockIO, 16384); // mode=0 => no copy block, 1-byte pre-transform length follows. // A zero pre-transform length is invalid and must be rejected. blockObs.writeBits(uint64(0), 8); blockObs.writeBits(uint64(0), 8); blockObs.close(); const string block = blockBuffer.str(); const uint64 blockBits = uint64(block.size()) << 3; stringbuf buffer; iostream io(&buffer); DefaultOutputBitStream obs(io, 16384); obs.writeBits(reinterpret_cast(header.data()), uint(header.size() << 3)); obs.writeBits(uint64(5), 5); // 3 + 5 = 8 bits to encode blockBits below obs.writeBits(blockBits, 8); obs.writeBits(reinterpret_cast(block.data()), uint(blockBits)); obs.close(); return buffer.str(); } static int expectHeaderFailure(const string& name, const string& data, int expectedError, const string& expectedText) { cout << "Test malformed header: " << name << endl; istringstream is(data); CompressedInputStream cis(is, 1); char dst[1]; try { cis.read(dst, 1); } catch (const IOException& e) { ASSERT_TRUE(e.error() == expectedError, "Unexpected IOException error code"); ASSERT_TRUE(string(e.what()).find(expectedText) != string::npos, "Unexpected IOException message"); return 0; } catch (const exception& e) { cerr << "Unexpected exception: " << e.what() << endl; return 1; } cerr << "Expected an exception for malformed header: " << name << endl; return 1; } static int expectBlockFailure(const string& name, const string& data, int expectedError, const string& expectedText, int jobs) { cout << "Test malformed block: " << name << endl; istringstream is(data); CompressedInputStream cis(is, jobs); char dst[1]; try { cis.read(dst, 1); } catch (const IOException& e) { ASSERT_TRUE(e.error() == expectedError, "Unexpected IOException error code"); ASSERT_TRUE(string(e.what()).find(expectedText) != string::npos, "Unexpected IOException message"); ASSERT_TRUE(cis.bad(), "Read failure did not set badbit"); return 0; } catch (const exception& e) { cerr << "Unexpected exception: " << e.what() << endl; return 1; } cerr << "Expected an exception for malformed block: " << name << endl; return 1; } int main() { const int version = 7; const int type = 0x4B414E5A; const int blockSize = 1024; const short entropy = EntropyDecoderFactory::ANS0_TYPE; const uint64 transform = uint64(TransformFactory::LZ_TYPE) << 42; if (expectHeaderFailure("invalid type", buildHeader(type ^ 1, version, 0, entropy, transform, blockSize, 0, 0, true), Error::ERR_INVALID_FILE, "Invalid stream type") != 0) { return 1; } if (expectHeaderFailure("unsupported version", buildHeader(type, version + 1, 0, entropy, transform, blockSize, 0, 0, true), Error::ERR_STREAM_VERSION, "cannot read this version") != 0) { return 1; } if (expectHeaderFailure("invalid checksum size", buildHeader(type, version, 3, entropy, transform, blockSize, 0, 0, true), Error::ERR_INVALID_FILE, "incorrect block checksum size") != 0) { return 1; } if (expectHeaderFailure("unknown entropy type", buildHeader(type, version, 0, 31, transform, blockSize, 0, 0, true), Error::ERR_INVALID_CODEC, "unknown entropy type") != 0) { return 1; } if (expectHeaderFailure("unknown transform type", buildHeader(type, version, 0, entropy, uint64(63) << 42, blockSize, 0, 0, true), Error::ERR_INVALID_CODEC, "unknown transform type") != 0) { return 1; } if (expectHeaderFailure("invalid block size", buildHeader(type, version, 0, entropy, transform, blockSize - 16, 0, 0, true), Error::ERR_BLOCK_SIZE, "incorrect block size") != 0) { return 1; } if (expectHeaderFailure("header checksum mismatch", buildHeader(type, version, 0, entropy, transform, blockSize, 0, 0, false), Error::ERR_CRC_CHECK, "header checksum mismatch") != 0) { return 1; } if (expectBlockFailure("zero pre-transform length", buildMalformedBlockStream(), Error::ERR_READ_FILE, "Invalid compressed block length", 1) != 0) { return 1; } if (expectBlockFailure("zero pre-transform length (multi-job)", buildMalformedBlockStream(), Error::ERR_READ_FILE, "Invalid compressed block length", 4) != 0) { return 1; } cout << "All malformed stream tests passed." << endl; return 0; } kanzi-cpp-2.6.0/src/test/TestTransforms.cpp000066400000000000000000001477331525576310500207130ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include "../types.hpp" #include "../util/strings.hpp" #include "../transform/AliasCodec.hpp" #include "../transform/BWT.hpp" #include "../transform/BWTS.hpp" #include "../transform/EXECodec.hpp" #include "../transform/FSDCodec.hpp" #include "../transform/LZCodec.hpp" #include "../transform/NullTransform.hpp" #include "../transform/RLT.hpp" #include "../transform/ROLZCodec.hpp" #include "../transform/SBRT.hpp" #include "../transform/SRT.hpp" #include "../transform/TextCodec.hpp" #include "../transform/TransformFactory.hpp" #include "../transform/UTFCodec.hpp" #include "../transform/ZRLT.hpp" using namespace std; using namespace kanzi; static const int BS_VERSION = 7; static void writeInt16LE(kanzi::byte buf[], int value) { buf[0] = kanzi::byte(value); buf[1] = kanzi::byte(value >> 8); } static void writeInt32LE(kanzi::byte buf[], int value) { buf[0] = kanzi::byte(value); buf[1] = kanzi::byte(value >> 8); buf[2] = kanzi::byte(value >> 16); buf[3] = kanzi::byte(value >> 24); } static vector createPEBlock(int arch) { const int size = 8192; const int codeStart = 512; const int codeLen = 4096; const int posPE = 0x80; vector data(size, kanzi::byte(0x90)); data[0] = kanzi::byte('M'); data[1] = kanzi::byte('Z'); writeInt32LE(&data[60], posPE); data[posPE] = kanzi::byte('P'); data[posPE + 1] = kanzi::byte('E'); data[posPE + 2] = kanzi::byte(0); data[posPE + 3] = kanzi::byte(0); writeInt16LE(&data[posPE + 4], arch); writeInt32LE(&data[posPE + 28], codeLen); writeInt32LE(&data[posPE + 44], codeStart); return data; } static void setPECodeLength(vector& data, int codeLen) { writeInt32LE(&data[0x80 + 28], codeLen); } static vector createELF64Block(int arch) { const int size = 8192; const int codeStart = 512; const int codeLen = 4096; const int posSection = 0x100; vector data(size, kanzi::byte(0)); data[0] = kanzi::byte(0x7F); data[1] = kanzi::byte('E'); data[2] = kanzi::byte('L'); data[3] = kanzi::byte('F'); data[4] = kanzi::byte(2); data[5] = kanzi::byte(1); writeInt16LE(&data[18], arch); writeInt16LE(&data[0x3A], 0x40); writeInt16LE(&data[0x3C], 1); writeInt32LE(&data[0x28], posSection); writeInt32LE(&data[posSection + 4], 1); writeInt32LE(&data[posSection + 0x18], codeStart); writeInt32LE(&data[posSection + 0x20], codeLen); return data; } static void fillX86Code(vector& data, int codeStart, int codeLen) { for (int i = codeStart; i + 5 <= codeStart + codeLen; i += 5) { data[i] = kanzi::byte(0xE8); data[i + 1] = kanzi::byte(0); data[i + 2] = kanzi::byte(0); data[i + 3] = kanzi::byte(0); data[i + 4] = kanzi::byte(0); } } static void fillARM64Code(vector& data, int codeStart, int codeLen) { for (int i = codeStart; i + 4 <= codeStart + codeLen; i += 4) writeInt32LE(&data[i], 0x14000000); } static void fillX86ExpandedCode(vector& data, int codeStart, int codeLen) { for (int i = codeStart; i + 8 <= codeStart + codeLen; i += 8) { const bool escaped = (((i - codeStart) >> 3) < 24); data[i] = kanzi::byte(0xE8); data[i + 1] = kanzi::byte(0); data[i + 2] = kanzi::byte(0); data[i + 3] = kanzi::byte(0); data[i + 4] = kanzi::byte(0); data[i + 5] = escaped ? kanzi::byte(0x9B) : kanzi::byte(0x90); data[i + 6] = kanzi::byte(0x90); data[i + 7] = kanzi::byte(0x90); } } static void addX86BoundaryJCC(vector& data, int codeStart, int codeLen) { const int idx = codeStart + codeLen - 5; data[idx] = kanzi::byte(0x0F); data[idx + 1] = kanzi::byte(0x85); data[idx + 2] = kanzi::byte(0); data[idx + 3] = kanzi::byte(0); data[idx + 4] = kanzi::byte(0); data[idx + 5] = kanzi::byte(0); } static vector createX86BoundaryBlock() { vector data = createPEBlock(0x014C); const int codeStart = 512; const int codeLen = 85; setPECodeLength(data, codeLen); fillX86Code(data, codeStart, 16 * 5); addX86BoundaryJCC(data, codeStart, codeLen); return data; } static int testEXERoundTrip(const string& name, vector& data) { cout << endl << "Correctness for " << name << endl; Context ctx; EXECodec codec(ctx); vector encoded(codec.getMaxEncodedLength(int(data.size())), kanzi::byte(0)); vector decoded(data.size(), kanzi::byte(0)); SliceArray input(&data[0], int(data.size()), 0); SliceArray output(&encoded[0], int(encoded.size()), 0); SliceArray reverse(&decoded[0], int(decoded.size()), 0); if (codec.forward(input, output, int(data.size())) == false) { cout << "Encoding error" << endl; return 1; } const int encodedSize = output._index; input._index = 0; output._index = 0; if (codec.inverse(output, reverse, encodedSize) == false) { cout << "Decoding error" << endl; return 1; } if ((reverse._index != int(data.size())) || (memcmp(&data[0], &decoded[0], data.size()) != 0)) { cout << "Round-trip mismatch" << endl; return 1; } vector small(encodedSize - 10, kanzi::byte(0)); SliceArray tooSmall(&small[0], int(small.size()), 0); output._index = 0; if (codec.inverse(output, tooSmall, encodedSize) != false) { cout << "Undersized output buffer should fail" << endl; return 1; } cout << "Identical" << endl; return 0; } static int testEXECodec() { vector x86 = createPEBlock(0x014C); fillX86Code(x86, 512, 4096); if (testEXERoundTrip("EXE-X86", x86) != 0) return 1; vector arm64 = createELF64Block(0x00B7); fillARM64Code(arm64, 512, 4096); if (testEXERoundTrip("EXE-ARM64", arm64) != 0) return 1; { cout << endl << "Correctness for EXE-X86-Expanded" << endl; Context ctx; EXECodec codec(ctx); vector expanded = createPEBlock(0x014C); fillX86ExpandedCode(expanded, 512, 4096); vector encoded(codec.getMaxEncodedLength(int(expanded.size())), kanzi::byte(0)); vector decoded(expanded.size(), kanzi::byte(0)); SliceArray input(&expanded[0], int(expanded.size()), 0); SliceArray output(&encoded[0], int(encoded.size()), 0); SliceArray reverse(&decoded[0], int(decoded.size()), 0); if (codec.forward(input, output, int(expanded.size())) == false) { cout << "Encoding error" << endl; return 1; } if (output._index <= int(expanded.size()) + 9) { cout << "Expected encoded block expansion beyond header" << endl; return 1; } const int encodedSize = output._index; output._index = 0; reverse._index = 0; if (codec.inverse(output, reverse, encodedSize) == false) { cout << "Decoding error" << endl; return 1; } if ((reverse._index != int(expanded.size())) || (memcmp(&expanded[0], &decoded[0], expanded.size()) != 0)) { cout << "Round-trip mismatch" << endl; return 1; } cout << "Identical" << endl; } vector boundary = createX86BoundaryBlock(); if (testEXERoundTrip("EXE-X86-Boundary-JCC", boundary) != 0) return 1; { cout << endl << "Correctness for EXE-X86-Legacy-Boundary-JCC" << endl; Context ctx; EXECodec codec(ctx); vector legacy = createX86BoundaryBlock(); vector encoded(codec.getMaxEncodedLength(int(legacy.size())), kanzi::byte(0)); vector decoded(legacy.size(), kanzi::byte(0)); SliceArray input(&legacy[0], int(legacy.size()), 0); SliceArray output(&encoded[0], int(encoded.size()), 0); SliceArray reverse(&decoded[0], int(decoded.size()), 0); if (codec.forward(input, output, int(legacy.size())) == false) { cout << "Encoding error" << endl; return 1; } const int encodedSize = output._index; const int codeEnd = LittleEndian::readInt32(&encoded[5]); if ((codeEnd >= encodedSize) || (encoded[codeEnd] != kanzi::byte(0x0F))) { cout << "Unexpected boundary layout" << endl; return 1; } writeInt32LE(&encoded[5], codeEnd + 1); output._index = 0; reverse._index = 0; if (codec.inverse(output, reverse, encodedSize) == false) { cout << "Decoding error" << endl; return 1; } if ((reverse._index != int(legacy.size())) || (memcmp(&legacy[0], &decoded[0], legacy.size()) != 0)) { cout << "Round-trip mismatch" << endl; return 1; } cout << "Identical" << endl; } return 0; } static int testTextCodecSelfDescribing() { cout << endl << "Correctness for TextCodec bitstream selector" << endl; string sample; for (int i = 0; i < 64; i++) { sample += "The quick brown fox jumps over the lazy dog. "; sample += "Text compression works best on repeated natural language content. "; sample += "This block verifies that the codec selector is carried in the stream header.\n"; } vector data(sample.size()); for (size_t i = 0; i < sample.size(); i++) data[i] = kanzi::byte(sample[i]); const int selectorMask = 0x10; for (int encType = 1; encType <= 2; encType++) { Context encCtx; encCtx.putInt("textcodec", encType); encCtx.putInt("bsVersion", 7); encCtx.putInt("blockSize", int(data.size())); TextCodec encoder(encCtx); vector encoded(encoder.getMaxEncodedLength(int(data.size())), kanzi::byte(0)); SliceArray input(&data[0], int(data.size()), 0); SliceArray output(&encoded[0], int(encoded.size()), 0); if (encoder.forward(input, output, int(data.size())) == false) { cout << "Encoding error for TextCodec" << encType << endl; return 1; } if ((((int(encoded[0]) & selectorMask) != 0) ? 2 : 1) != encType) { cout << "Invalid codec selector in TextCodec header" << endl; return 1; } Context decCtx; decCtx.putInt("textcodec", encType == 1 ? 2 : 1); decCtx.putInt("bsVersion", 7); TextCodec decoder(decCtx); vector decoded(data.size(), kanzi::byte(0)); // Keep the physical tail available to transforms that use guarded reads. SliceArray encodedInput(&encoded[0], int(encoded.size()), 0); SliceArray reverse(&decoded[0], int(decoded.size()), 0); if (decoder.inverse(encodedInput, reverse, output._index) == false) { cout << "Decoding error for TextCodec" << encType << endl; return 1; } if ((reverse._index != int(data.size())) || (memcmp(&data[0], &decoded[0], data.size()) != 0)) { cout << "Round-trip mismatch for TextCodec" << encType << endl; return 1; } } cout << "Identical" << endl; return 0; } static int testZRLTMalformed() { cout << endl << "Malformed ZRLT" << endl; Context ctx; ctx.putInt("bsVersion", BS_VERSION); ZRLT codec(ctx); { kanzi::byte encoded[1] = { kanzi::byte(2) }; kanzi::byte decoded[5]; memset(decoded, 0x7E, sizeof(decoded)); SliceArray input(encoded, 1, 0); SliceArray output(decoded, 4, 3); if (codec.inverse(input, output, 1) == false) { cout << "Valid offset decode failed" << endl; return 1; } if ((output._index != 4) || (decoded[3] != kanzi::byte(1)) || (decoded[4] != kanzi::byte(0x7E))) { cout << "Valid offset decode corrupted output" << endl; return 1; } } { kanzi::byte encoded[2] = { kanzi::byte(2), kanzi::byte(2) }; kanzi::byte decoded[5]; memset(decoded, 0x7E, sizeof(decoded)); SliceArray input(encoded, 2, 0); SliceArray output(decoded, 4, 3); if (codec.inverse(input, output, 2) != false) { cout << "Oversized offset decode should fail" << endl; return 1; } if (decoded[4] != kanzi::byte(0x7E)) { cout << "Oversized offset decode wrote past logical output" << endl; return 1; } } { kanzi::byte encoded[1] = { kanzi::byte(2) }; kanzi::byte decoded[1] = { kanzi::byte(0) }; SliceArray input(encoded, 1, 0); SliceArray output(decoded, 1, 0); if (codec.inverse(input, output, 2) != false) { cout << "Oversized encoded length should fail" << endl; return 1; } } { kanzi::byte encoded[1] = { kanzi::byte(0xFF) }; kanzi::byte decoded[1] = { kanzi::byte(0x7E) }; SliceArray input(encoded, 1, 0); SliceArray output(decoded, 1, 0); if (codec.inverse(input, output, 1) != false) { cout << "Truncated escape should fail" << endl; return 1; } if (decoded[0] != kanzi::byte(0x7E)) { cout << "Truncated escape wrote output" << endl; return 1; } } { kanzi::byte encoded[1] = { kanzi::byte(0) }; kanzi::byte decoded[1] = { kanzi::byte(0x7E) }; SliceArray input(encoded, 1, 0); SliceArray output(decoded, 0, 0); if (codec.inverse(input, output, 1) != false) { cout << "Oversized zero run should fail" << endl; return 1; } if (decoded[0] != kanzi::byte(0x7E)) { cout << "Oversized zero run wrote output" << endl; return 1; } } { kanzi::byte src[1] = { kanzi::byte(0xFE) }; kanzi::byte encoded[2] = { kanzi::byte(0x7E), kanzi::byte(0x7E) }; SliceArray input(src, 1, 0); SliceArray output(encoded, 1, 0); if (codec.forward(input, output, 1) != false) { cout << "Short escaped forward should fail" << endl; return 1; } if ((output._index != 0) || (encoded[1] != kanzi::byte(0x7E))) { cout << "Short escaped forward wrote past logical output" << endl; return 1; } } { kanzi::byte src[1] = { kanzi::byte(0) }; kanzi::byte encoded[1] = { kanzi::byte(0x7E) }; SliceArray input(src, 1, 0); SliceArray output(encoded, 1, 0); if (codec.forward(input, output, 1) == false) { cout << "Short zero run forward failed" << endl; return 1; } if ((output._index != 1) || (encoded[0] != kanzi::byte(0))) { cout << "Short zero run forward corrupted output" << endl; return 1; } } cout << "Malformed ZRLT tests passed" << endl; return 0; } static int testSRTMalformed() { cout << endl << "Malformed SRT" << endl; SRT codec; // The header declares 1025 bytes while the data area contains 1024. // This used to make SRT::inverse read one byte past the data area. const int headerSize = 257; const int dataSize = 1024; vector encoded(headerSize + dataSize, kanzi::byte(0)); encoded[0] = kanzi::byte(0xFF); // frequency[0] = 1023 encoded[1] = kanzi::byte(0x07); encoded[2] = kanzi::byte(2); // frequency[1] = 2 encoded[headerSize + 1] = kanzi::byte(1); encoded[headerSize + dataSize - 1] = kanzi::byte(1); vector decoded(dataSize, kanzi::byte(0x7E)); SliceArray input(&encoded[0], int(encoded.size()), 0); SliceArray output(&decoded[0], int(decoded.size()), 0); if (codec.inverse(input, output, int(encoded.size())) != false) { cout << "Inconsistent frequency table should fail" << endl; return 1; } for (size_t i = 0; i < decoded.size(); i++) { if (decoded[i] != kanzi::byte(0x7E)) { cout << "Malformed SRT input wrote output" << endl; return 1; } } if ((input._index != 0) || (output._index != 0)) { cout << "Malformed SRT input moved slice indexes" << endl; return 1; } cout << "Malformed SRT tests passed" << endl; return 0; } static int testROLZXMalformed() { cout << endl << "Malformed ROLZX" << endl; kanzi::byte encoded[13] = { kanzi::byte(0x00), kanzi::byte(0x00), kanzi::byte(0x02), kanzi::byte(0x00), kanzi::byte(0x00), kanzi::byte(0x00), kanzi::byte(0x7F), kanzi::byte(0xB1), kanzi::byte(0x47), kanzi::byte(0x07), kanzi::byte(0x91), kanzi::byte(0x2F), kanzi::byte(0xBF) }; kanzi::byte decoded[512]; memset(decoded, 0x7E, sizeof(decoded)); Context ctx; ctx.putString("transform", "ROLZX"); ROLZCodec codec(ctx); SliceArray input(encoded, int(sizeof(encoded)), 0); SliceArray output(decoded, int(sizeof(decoded)), 0); if (codec.inverse(input, output, int(sizeof(encoded))) != false) { cout << "Truncated ROLZX stream should fail" << endl; return 1; } cout << "Malformed ROLZX tests passed" << endl; return 0; } static int testTransformCapacityValidation() { cout << endl << "Transform capacity validation" << endl; Context ctx; ctx.putInt("bsVersion", BS_VERSION); kanzi::byte src[2] = { kanzi::byte(1), kanzi::byte(2) }; kanzi::byte dst[2] = { kanzi::byte(0x7E), kanzi::byte(0x7E) }; { BWT tf; SliceArray input(src, 1, 0); SliceArray output(dst, 2, 0); if (tf.forward(input, output, 2) != false) { cout << "BWT should reject oversized input count" << endl; return 1; } } { BWTS tf; SliceArray input(src, 2, 0); SliceArray output(dst, 0, 0); if (tf.forward(input, output, 1) != false) { cout << "BWTS should reject oversized output count" << endl; return 1; } } { SBRT tf(SBRT::MODE_RANK, ctx); SliceArray input(src, 2, 1); SliceArray output(dst, 2, 0); if (tf.forward(input, output, 2) != false) { cout << "SBRT should reject oversized remaining input count" << endl; return 1; } } { BWT tf; SliceArray input(src, 1, 0); SliceArray output(dst, 2, 0); if (tf.inverse(input, output, 2) != false) { cout << "BWT inverse should reject oversized input count" << endl; return 1; } } { BWTS tf; SliceArray input(src, 2, 0); SliceArray output(dst, 1, 0); if (tf.inverse(input, output, 2) != false) { cout << "BWTS inverse should reject oversized output count" << endl; return 1; } } { SBRT tf(SBRT::MODE_RANK, ctx); SliceArray input(src, 2, 0); SliceArray output(dst, 2, 0); const int savedIIdx = input._index; const int savedOIdx = output._index; if (tf.inverse(input, output, -1) != false) { cout << "SBRT inverse should reject negative counts" << endl; return 1; } if ((input._index != savedIIdx) || (output._index != savedOIdx)) { cout << "SBRT inverse negative count should not move indexes" << endl; return 1; } } { LZPCodec tf; kanzi::byte lzSrc[128]; kanzi::byte lzDst[145]; for (int i = 0; i < 128; i++) lzSrc[i] = kanzi::byte(i); memset(lzDst, 0x7E, sizeof(lzDst)); SliceArray input(lzSrc, 128, 0); SliceArray output(lzDst, 144, 1); const int savedIIdx = input._index; const int savedOIdx = output._index; if (tf.forward(input, output, 128) != false) { cout << "LZP forward should reject insufficient remaining output" << endl; return 1; } if ((input._index != savedIIdx) || (output._index != savedOIdx)) { cout << "LZP forward output capacity failure moved indexes" << endl; return 1; } } { LZPCodec tf; kanzi::byte lzSrc[128]; kanzi::byte lzDst[144]; for (int i = 0; i < 128; i++) lzSrc[i] = kanzi::byte(i); SliceArray input(lzSrc, 128, 1); SliceArray output(lzDst, 144, 0); const int savedIIdx = input._index; const int savedOIdx = output._index; if (tf.forward(input, output, 128) != false) { cout << "LZP forward should reject oversized remaining input count" << endl; return 1; } if ((input._index != savedIIdx) || (output._index != savedOIdx)) { cout << "LZP forward input capacity failure moved indexes" << endl; return 1; } } { LZXCodec tf; kanzi::byte lzSrc[128]; kanzi::byte lzEncoded[256]; // LZX inverse uses a 16-byte copy and requires trailing destination padding. kanzi::byte lzDecoded[128 + 16]; for (int i = 0; i < 128; i++) lzSrc[i] = kanzi::byte(i & 3); memset(lzEncoded, 0, sizeof(lzEncoded)); memset(lzDecoded, 0x7E, sizeof(lzDecoded)); SliceArray input(lzSrc, 128, 0); SliceArray encoded(lzEncoded, int(sizeof(lzEncoded)), 0); if (tf.forward(input, encoded, 128) == false) { cout << "LZX setup encoding failed" << endl; return 1; } const int encodedSize = encoded._index; SliceArray exactInput(lzEncoded, encodedSize, 0); SliceArray output(lzDecoded, int(sizeof(lzDecoded)), 0); if (tf.inverse(exactInput, output, encodedSize) != false) { cout << "LZX should reject input without the read-length guard" << endl; return 1; } if ((exactInput._index != 0) || (output._index != 0)) { cout << "LZX guard failure moved slice indexes" << endl; return 1; } SliceArray paddedInput(lzEncoded, int(sizeof(lzEncoded)), 0); if (tf.inverse(paddedInput, output, encodedSize) == false) { cout << "LZX should accept input with the read-length guard" << endl; return 1; } } cout << "Transform capacity validation passed" << endl; return 0; } static int testLZPMalformed() { cout << endl << "Malformed LZP" << endl; LZPCodec codec; { // A MATCH_FLAG at the end of the encoded block has no token byte. kanzi::byte encoded[6] = { kanzi::byte(0), kanzi::byte(0), kanzi::byte(0), kanzi::byte(0), kanzi::byte(0), kanzi::byte(0xFC) }; kanzi::byte decoded[128]; memset(decoded, 0x7E, sizeof(decoded)); SliceArray input(encoded, 6, 0); SliceArray output(decoded, 128, 0); if (codec.inverse(input, output, 6) != false) { cout << "Truncated LZP token should fail" << endl; return 1; } if ((input._index != 0) || (output._index != 0)) { cout << "Truncated LZP token moved slice indexes" << endl; return 1; } } { kanzi::byte encoded[4] = { kanzi::byte(0), kanzi::byte(0), kanzi::byte(0), kanzi::byte(0) }; kanzi::byte decoded[8]; memset(decoded, 0x7E, sizeof(decoded)); SliceArray input(encoded, 4, 0); SliceArray output(decoded, 3, 0); if (codec.inverse(input, output, 4) != false) { cout << "Short LZP output should fail" << endl; return 1; } if ((input._index != 0) || (output._index != 0) || (decoded[0] != kanzi::byte(0x7E)) || (decoded[1] != kanzi::byte(0x7E)) || (decoded[2] != kanzi::byte(0x7E))) { cout << "Short LZP output was modified" << endl; return 1; } } { kanzi::byte encoded[5] = { kanzi::byte(1), kanzi::byte(2), kanzi::byte(3), kanzi::byte(4), kanzi::byte(5) }; kanzi::byte decoded[8]; memset(decoded, 0x7E, sizeof(decoded)); SliceArray input(encoded, 5, 0); SliceArray output(decoded, 4, 0); if (codec.inverse(input, output, 5) != false) { cout << "LZP literal output overflow should fail" << endl; return 1; } if ((input._index != 0) || (output._index != 0) || (decoded[0] != kanzi::byte(0x7E)) || (decoded[1] != kanzi::byte(0x7E)) || (decoded[2] != kanzi::byte(0x7E)) || (decoded[3] != kanzi::byte(0x7E)) || (decoded[4] != kanzi::byte(0x7E))) { cout << "LZP literal output overflow modified output" << endl; return 1; } } cout << "Malformed LZP tests passed" << endl; return 0; } static int testUTFMalformed() { cout << endl << "Malformed UTF" << endl; // One-byte symbols are logically advanced by one byte, but the decoder // copies four bytes from every packed symbol. The fifth alias below is // at the end of the logical output buffer and must be rejected before // the four-byte copy. kanzi::byte encoded[16] = { kanzi::byte(0), kanzi::byte(0), kanzi::byte(0), kanzi::byte(1), kanzi::byte(0), kanzi::byte(0), kanzi::byte(0x41), kanzi::byte(0), kanzi::byte(0), kanzi::byte(0), kanzi::byte(0), kanzi::byte(0), kanzi::byte(0), kanzi::byte(0), kanzi::byte(0), kanzi::byte(0) }; kanzi::byte decoded[9]; memset(decoded, 0x7E, sizeof(decoded)); SliceArray input(encoded, 16, 0); SliceArray output(decoded, 5, 0); UTFCodec codec; if (codec.inverse(input, output, 16) != false) { cout << "UTF output overflow should fail" << endl; return 1; } for (int i = 5; i < int(sizeof(decoded)); i++) { if (decoded[i] != kanzi::byte(0x7E)) { cout << "UTF output overflow modified the canary" << endl; return 1; } } cout << "Malformed UTF tests passed" << endl; return 0; } static int testOverlappingTransformCopies() { cout << endl << "Overlapping transform copies" << endl; { NullTransform tf; kanzi::byte buf[6] = { kanzi::byte('a'), kanzi::byte('b'), kanzi::byte('c'), kanzi::byte('d'), kanzi::byte('e'), kanzi::byte('f') }; SliceArray input(buf, 4, 0); SliceArray output(buf, 5, 1); if (tf.forward(input, output, 4) == false) { cout << "NullTransform overlapping copy failed" << endl; return 1; } const kanzi::byte expected[6] = { kanzi::byte('a'), kanzi::byte('a'), kanzi::byte('b'), kanzi::byte('c'), kanzi::byte('d'), kanzi::byte('f') }; if ((memcmp(buf, expected, 6) != 0) || (input._index != 4) || (output._index != 5)) { cout << "NullTransform overlapping copy corrupted data" << endl; return 1; } } { Transform* transforms[8] = { new NullTransform(), nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr }; TransformSequence seq(transforms); seq.setSkipFlags(SKIP_MASK); kanzi::byte buf[6] = { kanzi::byte('a'), kanzi::byte('b'), kanzi::byte('c'), kanzi::byte('d'), kanzi::byte('e'), kanzi::byte('f') }; SliceArray input(buf, 4, 0); SliceArray output(buf, 5, 1); if (seq.inverse(input, output, 4) == false) { cout << "TransformSequence overlapping copy failed" << endl; return 1; } const kanzi::byte expected[6] = { kanzi::byte('a'), kanzi::byte('a'), kanzi::byte('b'), kanzi::byte('c'), kanzi::byte('d'), kanzi::byte('f') }; if ((memcmp(buf, expected, 6) != 0) || (input._index != 4) || (output._index != 5)) { cout << "TransformSequence overlapping copy corrupted data" << endl; return 1; } } cout << "Overlapping transform copies passed" << endl; return 0; } static int testFSDBucketedRoundTrip() { cout << endl << "FSD bucket layout round trip" << endl; const int dist = 4; const int bucketLength = 1 << 15; const int size = 2 * dist * bucketLength; vector data(size); for (int lane = 0; lane < dist; lane++) { int value = lane * 37 + 17; for (int sample = 0; sample < size / dist; sample++) { data[sample * dist + lane] = kanzi::byte(value); value += (((sample * 17 + lane * 13) % 5) == 0) ? 1 : 0; } } Context ctx; ctx.putInt("bsVersion", BS_VERSION); FSDCodec codec(ctx); vector encoded(codec.getMaxEncodedLength(size), kanzi::byte(0)); SliceArray input(&data[0], size, 0); SliceArray output(&encoded[0], int(encoded.size()), 0); if (codec.forward(input, output, size) == false) { cout << "Bucketed FSD encoding was skipped or failed" << endl; return 1; } if ((input._index != size) || (output._index != size + 2) || (int(output._array[1]) != dist) || ((int(output._array[0]) & 2) == 0)) { cout << "Invalid bucketed FSD header or size" << endl; return 1; } int outputIndex = 2 + dist; const int tileLength = dist * bucketLength; for (int tileStart = 0; tileStart < size; tileStart += tileLength) { const int tileEnd = min(tileStart + tileLength, size); for (int lane = 0; lane < dist; lane++) { const int firstPos = tileStart + lane + ((tileStart == 0) ? dist : 0); for (int pos = firstPos; pos < tileEnd; pos += dist) { kanzi::byte expected; if ((int(output._array[0]) & 1) == 0) { const uint residual = uint(uint8(int(data[pos]) - int(data[pos - dist]))); const uint zigzag = (residual & 0x80) ? ((256 - residual) << 1) - 1 : residual << 1; expected = kanzi::byte(zigzag); } else { expected = data[pos] ^ data[pos - dist]; } if (output._array[outputIndex++] != expected) { cout << "Bucketed FSD output order mismatch" << endl; return 1; } } } } if (outputIndex != output._index) { cout << "Bucketed FSD output length mismatch" << endl; return 1; } const int encodedSize = output._index; vector decoded(size, kanzi::byte(0)); SliceArray encodedInput(&encoded[0], int(encoded.size()), 0); SliceArray decodedOutput(&decoded[0], size, 0); if ((codec.inverse(encodedInput, decodedOutput, encodedSize) == false) || (encodedInput._index != encodedSize) || (decodedOutput._index != size)) { cout << "Bucketed FSD inverse failed" << endl; return 1; } for (int i = 0; i < size; i++) { if (data[i] != decoded[i]) { cout << "Bucketed FSD round trip mismatch" << endl; return 1; } } cout << "Bucketed FSD round trip passed" << endl; return 0; } static int testFSDLegacyInverse() { cout << endl << "FSD legacy inverse" << endl; const int encodedSize = 8; const int decodedSize = 6; kanzi::byte encoded[encodedSize] = { kanzi::byte(0), kanzi::byte(2), kanzi::byte(10), kanzi::byte(20), kanzi::byte(2), kanzi::byte(3), kanzi::byte(0), kanzi::byte(2) }; const kanzi::byte expected[decodedSize] = { kanzi::byte(10), kanzi::byte(20), kanzi::byte(11), kanzi::byte(18), kanzi::byte(11), kanzi::byte(19) }; kanzi::byte decoded[decodedSize] = { kanzi::byte(0) }; Context ctx; ctx.putInt("bsVersion", 6); FSDCodec codec(ctx); SliceArray input(encoded, encodedSize, 0); SliceArray output(decoded, decodedSize, 0); if ((codec.inverse(input, output, encodedSize) == false) || (input._index != encodedSize) || (output._index != decodedSize)) { cout << "FSD legacy inverse failed" << endl; return 1; } for (int i = 0; i < decodedSize; i++) { if (decoded[i] != expected[i]) { cout << "FSD legacy inverse mismatch" << endl; return 1; } } cout << "FSD legacy inverse passed" << endl; return 0; } static Transform* getByteTransform(string name, Context& ctx) { if (name.compare("SRT") == 0) return new SRT(ctx); if (name.compare("RLT") == 0) return new RLT(ctx); if (name.compare("ZRLT") == 0) return new ZRLT(ctx); if (name.compare("LZ") == 0) return new LZCodec(ctx); if (name.compare("LZX") == 0){ ctx.putInt("lz", TransformFactory::LZX_TYPE); return new LZCodec(ctx); } if (name.compare("LZP") == 0){ ctx.putInt("lz", TransformFactory::LZP_TYPE); return new LZCodec(ctx); } if (name.compare("ROLZ") == 0) return new ROLZCodec(ctx); if (name.compare("ROLZX") == 0) return new ROLZCodec(ctx); if (name.compare("RANK") == 0) return new SBRT(SBRT::MODE_RANK, ctx); if (name.compare("MTFT") == 0) return new SBRT(SBRT::MODE_MTF, ctx); if (name.compare("MM") == 0) return new FSDCodec(ctx); if (name.compare("NONE") == 0) return new NullTransform(ctx); if (name.compare("ALIAS") == 0) return new AliasCodec(ctx); cout << "No such byte transform: " << name << endl; return nullptr; } int testTransformsCorrectness(const string& name, int& errorIteration) { srand((uint)time(nullptr)); cout << endl << "Correctness for " << name << endl; int mod = (name == "ZRLT") ? 5 : 256; int res = 0; errorIteration = -1; for (int ii = 0; ii < 51; ii++) { cout << endl << "Test " << ii << endl; int size = 80000; // Declare size, will be updated in conditions vector values(1024 * 1024, kanzi::byte(0xAA)); if (name == "ALIAS") mod = 15 + 12 * ii; if (ii == 0) { size = 32; kanzi::byte arr[32] = { (kanzi::byte)0, (kanzi::byte)1, (kanzi::byte)2, (kanzi::byte)2, (kanzi::byte)2, (kanzi::byte)2, (kanzi::byte)7, (kanzi::byte)9, (kanzi::byte)9, (kanzi::byte)16, (kanzi::byte)16, (kanzi::byte)16, (kanzi::byte)1, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3, (kanzi::byte)3 }; memcpy(&values[0], &arr[0], size); } else if (ii < 10) { size = ii; memset(&values[0], ii, size); } else if (ii == 10) { size = 255; memset(&values[0], ii, size); values[127] = kanzi::byte(255); } else if (ii == 11) { size = 80000; kanzi::byte arr[80000]; arr[0] = kanzi::byte(1); for (int i = 1; i < 80000; i++) arr[i] = kanzi::byte(8); memcpy(&values[0], &arr[0], size); } else if (ii == 12) { size = 8; kanzi::byte arr[8] = { (kanzi::byte)0, (kanzi::byte)0, (kanzi::byte)1, (kanzi::byte)1, (kanzi::byte)2, (kanzi::byte)2, (kanzi::byte)3, (kanzi::byte)3 }; memcpy(&values[0], &arr[0], size); } else if (ii == 13) { // For RLT size = 512; kanzi::byte arr[512]; for (int i = 0; i < 256; i++) { arr[2 * i] = kanzi::byte(i); arr[2 * i + 1] = kanzi::byte(i); } arr[1] = kanzi::byte(255); // force RLT escape to be first symbol memcpy(&values[0], &arr[0], size); } else if (ii == 14) { // Lots of zeros size = 1024; kanzi::byte arr[1024] = { kanzi::byte(0) }; for (int i = 0; i < size; i++) { int val = rand() % 100; if (val >= 33) val = 0; arr[i] = kanzi::byte(val); } memcpy(&values[0], &arr[0], size); } else if (ii == 15) { // Lots of zeros size = 2048; kanzi::byte arr[2048] = { kanzi::byte(0) }; for (int i = 0; i < size; i++) { int val = rand() % 100; if (val >= 33) val = 0; arr[i] = kanzi::byte(val); } memcpy(&values[0], &arr[0], size); } else if (ii == 16) { // Totally random size = 512; kanzi::byte arr[512] = { kanzi::byte(0) }; // Leave zeros at the beginning for ZRLT to succeed for (int j = 20; j < 512; j++) arr[j] = kanzi::byte(rand() % mod); memcpy(&values[0], &arr[0], size); } else if (ii < 25) { size = 2048; kanzi::byte arr[2048] = { kanzi::byte(0) }; const int step = max(ii - 5, 2); arr[60] = kanzi::byte(rand() % mod); arr[61] = kanzi::byte(rand() % mod); arr[62] = kanzi::byte(rand() % mod); arr[63] = kanzi::byte(rand() % mod); // Simulate interleaved channels for MM for (int j = 64; j + step < size; j += step) { for (int k = 0; k < step; k++) arr[j + k] = arr[j + k - step]; } memcpy(&values[0], &arr[0], size); } else if (ii == 50) { cout << "Large random data" << endl; size = 1024 * 1024; kanzi::byte* arr = new kanzi::byte[size]; for (int i = 0; i < size; i++) arr[i] = kanzi::byte(rand() % 256); memcpy(&values[0], arr, size); delete[] arr; } else { size = 1024; kanzi::byte arr[1024] = { kanzi::byte(0) }; // Leave zeros at the beginning for ZRLT to succeed int idx = 20; while (idx < 1024) { int len = rand() % 120; // above LZP min match threshold if (len % 3 == 0) len = 1; kanzi::byte val = kanzi::byte(rand() % mod); int end = (idx + len) < size ? idx + len : size; for (int j = idx; j < end; j++) arr[j] = val; idx += len; } memcpy(&values[0], &arr[0], size); } Context ctx; ctx.putInt("bsVersion", BS_VERSION); ctx.putString("transform", name); Transform* ff = getByteTransform(name, ctx); if (ff == nullptr) { errorIteration = ii; return 1; } Transform* fi = getByteTransform(name, ctx); if (fi == nullptr) { delete ff; errorIteration = ii; return 1; } const int dstSize = ff->getMaxEncodedLength(size); const int reverseSize = size + 16; kanzi::byte* input = new kanzi::byte[size]; kanzi::byte* output = new kanzi::byte[dstSize]; kanzi::byte* reverse = new kanzi::byte[reverseSize]; SliceArray iba1(input, size, 0); SliceArray iba2(output, dstSize, 0); SliceArray iba3(reverse, reverseSize, 0); memset(output, 0xAA, dstSize); memset(reverse, 0xAA, reverseSize); int count; for (int i = 0; i < size; i++) input[i] = values[i]; cout << endl << "Original: " << endl; if (ii == 11) { cout << "1 8 (" << (size - 1) << " times)"; } else { if (size > 1024) { cout << "Large data block - not printing all values."; } else { for (int i = 0; i < size; i++) cout << (int(input[i]) & 0xFF) << " "; } } if (ff->forward(iba1, iba2, size) == false) { if ((iba1._index != size) || (iba2._index >= iba1._index)) { cout << endl << "No compression (ratio > 1.0), skip reverse" << endl; delete ff; delete fi; delete[] input; delete[] output; delete[] reverse; continue; } cout << endl << "Encoding error" << endl; errorIteration = ii; res = 1; goto End; } if (name != "MM") { // MM can expand if ((iba1._index != size) || (iba1._index < iba2._index)) { cout << endl << "No compression (ratio > 1.0), skip reverse" << endl; delete ff; delete fi; delete[] input; delete[] output; delete[] reverse; continue; } } cout << endl; cout << "Coded: " << endl; if (iba2._index > 1024) { cout << "Large data block - not printing all values."; } else { for (int i = 0; i < iba2._index; i++) cout << (int(output[i]) & 0xFF) << " "; } cout << " (Compression ratio: " << (iba2._index * 100 / size) << "%)" << endl; count = iba2._index; iba1._index = 0; iba2._index = 0; iba3._index = 0; if (fi->inverse(iba2, iba3, count) == false) { cout << "Decoding error" << endl; errorIteration = ii; res = 1; goto End; } cout << "Decoded: " << endl; if (ii == 11) { cout << "1 8 (" << (size - 1) << " times)"; } else { if (size > 1024) { cout << "Large data block - not printing all values."; } else { for (int i = 0; i < size; i++) cout << (int(reverse[i]) & 0xFF) << " "; } } cout << endl; for (int i = 0; i < size; i++) { if (input[i] != reverse[i]) { cout << "Different (index " << i << ": "; cout << (int(input[i]) & 0xFF) << " - " << (int(reverse[i]) & 0xFF); cout << ")" << endl; errorIteration = ii; res = 1; goto End; } } cout << endl << "Identical" << endl << endl; End: if (ff != nullptr) delete ff; if (fi != nullptr) delete fi; delete[] input; delete[] output; delete[] reverse; } return res; } int testTransformsSpeed(const string& name) { // Test speed srand((uint)time(nullptr)); int iter = 50000; if ((name == "ROLZ") || (name == "SRT") || (name == "RANK") || (name == "MTFT")) iter = 4000; int size = 30000; int res = 0; cout << endl << endl << "Speed test for " << name << endl; cout << "Iterations: " << iter << endl; cout << endl; kanzi::byte input[50000] = { kanzi::byte(0) }; kanzi::byte output[50000] = { kanzi::byte(0) }; kanzi::byte reverse[50000] = { kanzi::byte(0) }; Context ctx; ctx.putInt("bsVersion", BS_VERSION); Transform* f = getByteTransform(name, ctx); if (f == nullptr) return -1; SliceArray iba1(input, size, 0); SliceArray iba2(output, f->getMaxEncodedLength(size), 0); const int reverseSize = size + 16; SliceArray iba3(reverse, reverseSize, 0); int mod = (name == "ZRLT") ? 5 : 256; delete f; for (int jj = 0; jj < 3; jj++) { // Generate random data with runs // Leave zeros at the beginning for ZRLT to succeed int n = iter / 20; if (name == "ALIAS") mod = 5 + 80 * jj; while (n < size) { kanzi::byte val = kanzi::byte(rand() % mod); input[n++] = val; int run = rand() % 256; run -= 220; while ((--run > 0) && (n < size)) input[n++] = val; } clock_t before, after; double delta1 = 0; double delta2 = 0; for (int ii = 0; ii < iter; ii++) { Transform* ff = getByteTransform(name, ctx); if (ff == nullptr) return -1; iba1._index = 0; iba2._index = 0; before = clock(); if (ff->forward(iba1, iba2, size) == false) { if ((iba1._index != size) || (iba2._index >= iba1._index)) { cout << endl << "No compression (ratio > 1.0), skip reverse" << endl; delete ff; continue; } cout << "Encoding error" << endl; delete ff; return -1; } after = clock(); delta1 += (after - before); delete ff; } int count = iba2._index; for (int ii = 0; ii < iter; ii++) { Transform* fi = getByteTransform(name, ctx); if (fi == nullptr) return -1; iba3._index = 0; iba2._index = 0; before = clock(); if (fi->inverse(iba2, iba3, count) == false) { cout << "Decoding error" << endl; delete fi; return -1; } after = clock(); delta2 += (after - before); delete fi; } int idx = -1; // Sanity check for (int i = 0; i < iba1._index; i++) { if (iba1._array[i] != iba3._array[i]) { idx = i; break; } } if (idx >= 0) { cout << "Failure at index " << idx << " (" << (int)iba1._array[idx]; cout << "<->" << (int)iba3._array[idx] << ")" << endl; res = -1; } // MB = 1000 * 1000, MiB = 1024 * 1024 double prod = double(iter) * double(size); double b2MiB = double(1) / double(1024 * 1024); double d1_sec = delta1 / CLOCKS_PER_SEC; double d2_sec = delta2 / CLOCKS_PER_SEC; cout << name << " encoding [ms]: " << (int)(d1_sec * 1000) << endl; cout << "Throughput [MiB/s]: " << (int)(prod * b2MiB / d1_sec) << endl; cout << name << " decoding [ms]: " << (int)(d2_sec * 1000) << endl; cout << "Throughput [MiB/s]: " << (int)(prod * b2MiB / d2_sec) << endl; } return res; } #ifdef __GNUG__ int main(int argc, const char* argv[]) #else int TestTransforms_main(int argc, const char* argv[]) #endif { int res = 0; int errorIteration = -1; string errCodec; try { res = testTextCodecSelfDescribing(); if (res != 0) return res; res = testEXECodec(); if (res != 0) return res; res = testZRLTMalformed(); if (res != 0) return res; res = testSRTMalformed(); if (res != 0) return res; res = testROLZXMalformed(); if (res != 0) return res; res = testLZPMalformed(); if (res != 0) return res; res = testUTFMalformed(); if (res != 0) return res; res = testTransformCapacityValidation(); if (res != 0) return res; res = testOverlappingTransformCopies(); if (res != 0) return res; res = testFSDBucketedRoundTrip(); if (res != 0) return res; res = testFSDLegacyInverse(); if (res != 0) return res; vector codecs; bool doPerf = true; if (argc == 1) { #if __cplusplus < 201103L string allCodecs[13] = { "LZ", "LZX", "LZP", "ROLZ", "ROLZX", "RLT", "ZRLT", "RANK", "SRT", "NONE", "ALIAS", "MM", "MTFT" }; for (int i = 0; i < 13; i++) codecs.push_back(allCodecs[i]); #else codecs = { "LZ", "LZX", "LZP", "ROLZ", "ROLZX", "RLT", "ZRLT", "RANK", "SRT", "NONE", "ALIAS", "MM", "MTFT" }; #endif } else { string str = argv[1]; transform(str.begin(), str.end(), str.begin(), safeToUpper); if (str != "-TYPE=ALL") { codecs.push_back(str.substr(6)); if (str.compare(0, 6, "-TYPE=") != 0) { cout << "Missing transform type" << endl; return 1; } } else { #if __cplusplus < 201103L string allCodecs[13] = { "LZ", "LZX", "LZP", "ROLZ", "ROLZX", "RLT", "ZRLT", "RANK", "SRT", "NONE", "ALIAS", "MM", "MTFT" }; for (int i = 0; i < 13; i++) codecs.push_back(allCodecs[i]); #else codecs = { "LZ", "LZX", "LZP", "ROLZ", "ROLZX", "RLT", "ZRLT", "RANK", "SRT", "NONE", "ALIAS", "MM", "MTFT" }; #endif } if (argc > 2) { str = argv[2]; transform(str.begin(), str.end(), str.begin(), safeToUpper); doPerf = str != "-NOPERF"; } } for (vector::iterator it = codecs.begin(); it != codecs.end(); ++it) { errCodec = *it; cout << endl << endl << "Test" << *it << endl; res = testTransformsCorrectness(*it, errorIteration); if (res) break; if ((doPerf == true) && (*it != "LZP") && (*it != "MM")) { // skip codecs with no good data res = testTransformsSpeed(*it); if (res) break; } } } catch (const exception& e) { cout << e.what() << endl; res = 123; } cout << endl; if (res == 0) { cout << "Success" << endl; } else { cout << "Failure"; if (res < 0) { cout << " in benchmarking"; } else if (errorIteration >= 0) { cout << " in test " << errorIteration; } if (errCodec.empty() == false) cout << " (" << errCodec << ")"; cout << endl; } return res; } kanzi-cpp-2.6.0/src/test/test_api.py000066400000000000000000000132371525576310500173620ustar00rootroot00000000000000import os import tempfile from kanzi import Compressor, Decompressor, KanziError # ----------------------------------------------------------------------------- # Utilities # ----------------------------------------------------------------------------- def make_params(): return dict( transform=b"LZX", entropy=b"HUFFMAN", block_size=1024, jobs=1, checksum=0, headerless=0, ) def fill_buffer(size): return bytes((i * 17 + 3) & 0xFF for i in range(size)) def write_file(path, data: bytes): with open(path, "wb") as f: f.write(data) # ----------------------------------------------------------------------------- # Tests # ----------------------------------------------------------------------------- def test_init_invalid(): print("TEST: initCompressor invalid params...") # Our Python wrapper validates parameters eagerly, # so most invalid cases are raised as Python exceptions. try: Compressor( dst_path="/dev/null", transform=None, # invalid ) except Exception: pass else: raise AssertionError("init should fail on invalid transform") def test_init_dispose(): print("TEST: init + dispose...") with tempfile.NamedTemporaryFile(delete=True) as tmp: with Compressor(tmp.name, **make_params()) as c: assert c is not None def test_compress_small(): print("TEST: small compression...") data = fill_buffer(256) with tempfile.NamedTemporaryFile(delete=True) as tmp: with Compressor(tmp.name, **make_params()) as c: written = c.compress(data) assert written >= 0 def test_compress_too_big(): print("TEST: oversized block handling...") params = make_params() params["block_size"] = 1024 big = fill_buffer(4096) with tempfile.NamedTemporaryFile(delete=True) as tmp: with Compressor(tmp.name, **params) as c: try: c.compress(big) assert False, "compress should fail on oversized input" except KanziError: pass def test_compress_two_blocks(): print("TEST: two-block compression...") params = make_params() params["block_size"] = 1024 a = fill_buffer(300) b = fill_buffer(500) with tempfile.NamedTemporaryFile(delete=True) as tmp: with Compressor(tmp.name, **params) as c: out1 = c.compress(a) out2 = c.compress(b) assert out1 >= 0 assert out2 >= 0 def test_basic_decompression(): print("TEST: basic decompression...") input_data = b"Hello Kanzi! Hello Compression!" with tempfile.NamedTemporaryFile(delete=False) as comp: comp_name = comp.name # Compress with Compressor( comp_name, transform=b"LZ", entropy=b"ANS0", block_size=1 << 16, jobs=1, checksum=32, headerless=0, ) as c: c.compress(input_data) # Decompress with Decompressor( comp_name, buffer_size=1 << 16, jobs=1, headerless=0, ) as d: out = d.decompress_block(1024) assert out == input_data, "decompressed data mismatch" os.remove(comp_name) def test_large_multi_block(): print("TEST: large multi blocks") size = 2 * 1024 * 1024 data = bytes((i * 7) & 0xFF for i in range(size)) with tempfile.NamedTemporaryFile(delete=False) as comp: comp_name = comp.name # Compress with Compressor( comp_name, transform=b"LZ", entropy=b"FPAQ", block_size=256 * 1024, jobs=1, checksum=64, headerless=0, ) as c: offset = 0 while offset < size: chunk = min(256 * 1024, size - offset) c.compress(data[offset:offset + chunk]) offset += chunk # Decompress out = bytearray() with Decompressor( comp_name, buffer_size=256 * 1024, jobs=1, headerless=0, ) as d: while True: try: block = d.decompress_block(256 * 1024) if not block: break out.extend(block) except KanziError: break # expected EOF assert bytes(out) == data, "large decompression mismatch" os.remove(comp_name) def test_headerless(): print("TEST: headerless") input_data = b"HEADERLESS MODE IS ACTIVE" with tempfile.NamedTemporaryFile(delete=False) as comp: comp_name = comp.name # Compress (headerless) with Compressor( comp_name, transform=b"LZ", entropy=b"ANS0", block_size=1 << 15, jobs=1, checksum=0, headerless=1, ) as c: c.compress(input_data) # Decompress (headerless) with Decompressor( comp_name, buffer_size=1 << 15, jobs=1, headerless=1, transform=b"LZ", entropy=b"ANS0", blockSize=1 << 15, originalSize=len(input_data), checksum=0, bsVersion=1, ) as d: out = d.decompress_block(256) assert out == input_data, "headerless decompression mismatch" os.remove(comp_name) # ----------------------------------------------------------------------------- # Main # ----------------------------------------------------------------------------- def main(): test_init_invalid() test_init_dispose() test_compress_small() test_compress_too_big() test_compress_two_blocks() test_basic_decompression() test_large_multi_block() test_headerless() print("All Python API tests passed.") if __name__ == "__main__": main() kanzi-cpp-2.6.0/src/test/test_roundtrip.py000066400000000000000000000201101525576310500206230ustar00rootroot00000000000000#!/usr/bin/env python3 import argparse import hashlib import shutil import shlex import subprocess import sys from pathlib import Path LANG_TO_COMMAND = { "cpp": "/Users/flanglet/ws/kanzi-cpp/bin/kanzi", "java": "java -jar /Users/flanglet/ws/kanzi/java/target/kanzi-2.5.0.jar", "go": "/Users/flanglet/ws/kanzi-go/v2/app/Kanzi", } def md5_file(path: Path) -> str: hsh = hashlib.md5() with path.open("rb") as f: while True: chunk = f.read(1024 * 1024) if not chunk: break hsh.update(chunk) return hsh.hexdigest() def md5_dir(path: Path) -> str: hsh = hashlib.md5() files = [p for p in path.rglob("*") if p.is_file()] files.sort(key=lambda p: str(p.relative_to(path))) for file_path in files: rel = str(file_path.relative_to(path)).replace("\\", "/") hsh.update(rel.encode("utf-8")) hsh.update(b"\0") with file_path.open("rb") as f: while True: chunk = f.read(1024 * 1024) if not chunk: break hsh.update(chunk) return hsh.hexdigest() def list_relative_files(path: Path) -> list[Path]: files = [p.relative_to(path) for p in path.rglob("*") if p.is_file()] files.sort(key=lambda p: str(p).replace("\\", "/")) return files def clean_dir(path: Path) -> None: if path.exists(): shutil.rmtree(path) path.mkdir(parents=True, exist_ok=True) def run_command(argv: list[str]) -> None: print("Running:", " ".join(shlex.quote(x) for x in argv)) lines: list[str] = [] with subprocess.Popen( argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True ) as proc: assert proc.stdout is not None for line in proc.stdout: lines.append(line) stripped = line.lstrip() if stripped.startswith("Compressed") or stripped.startswith("Decompressed"): print(stripped.rstrip("\n")) rc = proc.wait() if rc != 0: if lines: print("Command output:", file=sys.stderr) for line in lines: print(line.rstrip("\n"), file=sys.stderr) raise subprocess.CalledProcessError(rc, argv) def resolve_command(language: str | None, command: str | None) -> str: if language is not None: return LANG_TO_COMMAND[language] if command is None or not command.strip(): raise ValueError( "Empty command: provide --language (cpp|java|go) or a non-empty --command" ) return command.strip() def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="Test compression/decompression round trip with MD5 validation." ) parser.add_argument("level", help="Compression level") parser.add_argument("input_path", help="Input file or directory path") parser.add_argument( "--block-size", dest="block_size", default=None, help="Optional block size (passed to -b)", ) parser.add_argument( "--command", default=None, help="Kanzi command to run (ignored when --language is provided)", ) parser.add_argument( "--language", choices=sorted(LANG_TO_COMMAND.keys()), default=None, help="Select command automatically based on language", ) parser.add_argument( "--verbose", default="1", help="Kanzi verbosity level (default: 1)", ) parser.add_argument( "--show-only-mismatch", action="store_true", help="For directory input, only display files with checksum mismatch", ) return parser def main() -> int: args = build_parser().parse_args() input_path = Path(args.input_path).resolve() if not input_path.exists(): print(f"Error: input path does not exist: {input_path}", file=sys.stderr) return 2 try: command = resolve_command(args.language, args.command) except ValueError as exc: print(f"Error: {exc}", file=sys.stderr) return 2 base_cmd = shlex.split(command) tmp_root = Path("/tmp") try: if input_path.is_dir(): out_path = tmp_root / "out" bak_path = tmp_root / "bak" clean_dir(out_path) clean_dir(bak_path) cmp_input = input_path cmp_output = out_path dcmp_input = out_path dcmp_output = bak_path cmp_target = input_path dcmp_target = bak_path else: out_path = tmp_root / (input_path.name + ".knz") bak_path = tmp_root / (input_path.name + ".bak") if out_path.exists(): out_path.unlink() if bak_path.exists(): bak_path.unlink() cmp_input = input_path cmp_output = out_path dcmp_input = out_path dcmp_output = bak_path cmp_target = input_path dcmp_target = bak_path compress_cmd = ( base_cmd + [ "-c", "-i", str(cmp_input), "-o", str(cmp_output), "-l", str(args.level), "-f", "-v", str(args.verbose), ] ) if args.block_size is not None: compress_cmd += ["-b", str(args.block_size)] decompress_cmd = ( base_cmd + [ "-d", "-i", str(dcmp_input), "-o", str(dcmp_output), "-f", "-v", str(args.verbose), ] ) run_command(compress_cmd) run_command(decompress_cmd) if cmp_target.is_dir(): orig_files = set(list_relative_files(cmp_target)) dec_files = set(list_relative_files(dcmp_target)) all_files = sorted(orig_files | dec_files, key=lambda p: str(p).replace("\\", "/")) all_match = True for rel in all_files: rel_str = str(rel).replace("\\", "/") orig_abs = cmp_target / rel dec_abs = dcmp_target / rel if rel not in orig_files: all_match = False print(f"[MISMATCH] {rel_str}: missing in original, present in decompressed") continue if rel not in dec_files: all_match = False print(f"[MISMATCH] {rel_str}: present in original, missing in decompressed") continue md5_original = md5_file(orig_abs) md5_roundtrip = md5_file(dec_abs) status = "OK" if md5_original == md5_roundtrip else "MISMATCH" if status == "MISMATCH": all_match = False if (not args.show_only_mismatch) or status == "MISMATCH": print( f"[{status}] {rel_str}: Original MD5={md5_original} " f"Decompressed MD5={md5_roundtrip}" ) print("Success" if all_match else "Failure") return 0 if all_match else 1 else: md5_original = md5_file(cmp_target) md5_roundtrip = md5_file(dcmp_target) match = md5_original == md5_roundtrip if (not args.show_only_mismatch) or (not match): status = "OK" if match else "MISMATCH" print( f"[{status}] {cmp_target.name}: Original MD5={md5_original} " f"Decompressed MD5={md5_roundtrip}" ) print("Success" if match else "Failure") return 0 if match else 1 except subprocess.CalledProcessError as exc: print(f"Error: command failed with exit code {exc.returncode}", file=sys.stderr) return exc.returncode except Exception as exc: print(f"Error: {exc}", file=sys.stderr) return 1 if __name__ == "__main__": sys.exit(main()) kanzi-cpp-2.6.0/src/transform/000077500000000000000000000000001525576310500162265ustar00rootroot00000000000000kanzi-cpp-2.6.0/src/transform/AliasCodec.cpp000066400000000000000000000276651525576310500207410ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include "AliasCodec.hpp" #include "../Global.hpp" #include "../Memory.hpp" using namespace kanzi; using namespace std; const int AliasCodec::MIN_BLOCK_SIZE = 1024; AliasCodec::AliasCodec(Context& ctx) : _pCtx(&ctx) { _onlyDNA = _pCtx->getInt("packOnlyDNA", 0) != 0; } bool AliasCodec::forward(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (count < MIN_BLOCK_SIZE) return false; if (!SliceArray::isValid(input)) throw invalid_argument("Alias codec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("Alias codec: Invalid output block"); if (output._length - output._index < getMaxEncodedLength(count)) return false; Global::DataType dt = Global::UNDEFINED; if (_pCtx != nullptr) { dt = (Global::DataType) _pCtx->getInt("dataType", Global::UNDEFINED); if ((dt == Global::MULTIMEDIA) || (dt == Global::UTF8)) return false; if ((dt == Global::EXE) || (dt == Global::BIN)) return false; if ((_onlyDNA == true) && (dt != Global::UNDEFINED) && (dt != Global::DNA)) return false; } kanzi::byte* dst = &output._array[output._index]; const kanzi::byte* src = &input._array[input._index]; // Find missing 1-kanzi::byte symbols uint freqs0[256] = { 0 }; Global::computeHistogram(&src[0], count, freqs0, true); int n0 = 0; int absent[256] = { 0 }; for (int i = 0; i < 256; i++) { if (freqs0[i] == 0) absent[n0++] = i; } if (n0 < 16) return false; if (dt == Global::UNDEFINED) { dt = Global::detectSimpleType(count, freqs0); if ((_pCtx != nullptr) && (dt != Global::UNDEFINED)) _pCtx->putInt("dataType", dt); if ((dt != Global::DNA) && (_onlyDNA == true)) return false; } int srcIdx, dstIdx; if (n0 >= 240) { // Small alphabet => pack bits dst[0] = kanzi::byte(n0); dstIdx = 1; if (n0 == 255) { // One symbol dst[1] = src[0]; LittleEndian::writeInt32(&dst[2], count); dstIdx = 6; srcIdx = count; } else { srcIdx = 0; kanzi::byte map8[256] = { kanzi::byte(0) }; for (int i = 0, j = 0; i < 256; i++) { if (freqs0[i] != 0) { dst[dstIdx++] = kanzi::byte(i); map8[i] = kanzi::byte(j++); } } if (n0 >= 252) { // 4 symbols or less const int c3 = count & 3; dst[dstIdx++] = kanzi::byte(c3); memcpy(&dst[dstIdx], &src[srcIdx], size_t(c3)); srcIdx += c3; dstIdx += c3; while (srcIdx < count) { dst[dstIdx++] = (map8[int(src[srcIdx + 0])] << 6) | (map8[int(src[srcIdx + 1])] << 4) | (map8[int(src[srcIdx + 2])] << 2) | map8[int(src[srcIdx + 3])]; srcIdx += 4; } } else { // 16 symbols or less dst[dstIdx++] = kanzi::byte(count & 1); if ((count & 1) != 0) dst[dstIdx++] = src[srcIdx++]; while (srcIdx < count) { dst[dstIdx++] = (map8[int(src[srcIdx])] << 4) | map8[int(src[srcIdx + 1])]; srcIdx += 2; } } } } else { // Digram encoding vector v; { // Find missing 2-kanzi::byte symbols uint* freqs1 = new uint[65536]; memset(freqs1, 0, 65536 * sizeof(uint)); Global::computeHistogram(&src[0], count, freqs1, false); int n1 = 0; for (uint32 i = 0; i < 65536; i++) { if (freqs1[i] == 0) continue; #if __cplusplus >= 201103L v.emplace_back(i, freqs1[i]); #else sdAlias a(i, freqs1[i]); v.push_back(a); #endif n1++; } delete[] freqs1; if (n1 < n0) { // Fewer distinct 2-kanzi::byte symbols than 1-kanzi::byte symbols n0 = n1; if (n0 < 16) return false; } // Only the n0 most frequent digrams are used, in decreasing order. partial_sort(v.begin(), v.begin() + n0, v.end()); } int16 map16[65536]; // Build map symbol -> alias for (int i = 0; i < 65536; i++) map16[i] = 0x100 | int16(i >> 8); int savings = 0; dst[0] = kanzi::byte(n0); dst[1] = kanzi::byte(0); srcIdx = 0; dstIdx = 2; // Header: emit map data for (int i = 0; i < n0; i++) { const sdAlias& sd = v[i]; savings += sd.freq; // ignore factor 2 const int idx = sd.val; map16[idx] = int16(absent[i]) | 0x200; dst[dstIdx] = kanzi::byte(idx >> 8); dst[dstIdx + 1] = kanzi::byte(idx); dst[dstIdx + 2] = kanzi::byte(absent[i]); dstIdx += 3; } // Worth it? if (savings < count / 20) return false; v.clear(); const int srcEnd = count - 1; // Emit aliased data while (srcIdx < srcEnd) { const int16 alias = map16[(int(src[srcIdx]) << 8) | int(src[srcIdx + 1])]; dst[dstIdx++] = kanzi::byte(alias); srcIdx += (alias >> 8); } if (srcIdx != count) { dst[1] = kanzi::byte(1); dst[dstIdx++] = src[srcIdx++]; } } input._index += srcIdx; output._index += dstIdx; return dstIdx < count; } bool AliasCodec::inverse(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (!SliceArray::isValid(input)) throw invalid_argument("Alias codec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("Alias codec: Invalid output block"); if (input._index + count > input._length) return false; kanzi::byte* dst = &output._array[output._index]; const kanzi::byte* src = &input._array[input._index]; const int dstEnd = output._length - output._index; int n = int(src[0]); if (n < 16) return false; int srcIdx; int dstIdx = 0; if (n >= 240) { n = 256 - n; srcIdx = 1; if (n == 1) { // One symbol if (count < 6) return false; const kanzi::byte val = src[1]; const int oSize = LittleEndian::readInt32(&src[2]); if ((oSize < 0) || (oSize > dstEnd)) return false; memset(&dst[0], int(val), size_t(oSize)); srcIdx = count; dstIdx = oSize; } else { // Rebuild map alias -> symbol kanzi::byte idx2symb[16] = { kanzi::byte(0) }; if (srcIdx + n + 1 > count) return false; for (int i = 0; i < n; i++) idx2symb[i] = src[srcIdx++]; const int adjust = int(src[srcIdx++]); if ((adjust < 0) || (adjust >= 4)) return false; if (n <= 4) { // 4 symbols or less int32 decodeMap[256] = { 0 }; for (int i = 0; i < 256; i++) { int32 val; val = int32(idx2symb[(i >> 0) & 0x03]); val <<= 8; val |= int32(idx2symb[(i >> 2) & 0x03]); val <<= 8; val |= int32(idx2symb[(i >> 4) & 0x03]); val <<= 8; val |= int32(idx2symb[(i >> 6) & 0x03]); decodeMap[i] = val; } if ((srcIdx + adjust > count) || (dstIdx + adjust > dstEnd)) return false; memcpy(&dst[dstIdx], &src[srcIdx], size_t(adjust)); srcIdx += adjust; dstIdx += adjust; if ((count - srcIdx) > ((dstEnd - dstIdx) >> 2)) return false; while (srcIdx < count) { LittleEndian::writeInt32(&dst[dstIdx], decodeMap[int(src[srcIdx++])]); dstIdx += 4; } } else { // 16 symbols or less int16 decodeMap[256] = { 0 }; for (int i = 0; i < 256; i++) { int16 val = int16(idx2symb[i& 0x0F]); val <<= 8; val |= int16(idx2symb[i >> 4]); decodeMap[i] = val; } if (adjust != 0) { if ((srcIdx >= count) || (dstIdx >= dstEnd)) return false; dst[dstIdx++] = src[srcIdx++]; } if ((count - srcIdx) > ((dstEnd - dstIdx) >> 1)) return false; while (srcIdx < count) { LittleEndian::writeInt16(&dst[dstIdx], decodeMap[int(src[srcIdx++])]); dstIdx += 2; } } } } else { if (count < 2) return false; // Rebuild map alias -> symbol int map16[256] = { 0 }; const int adjust = int(src[1]); if ((adjust < 0) || (adjust > 1)) return false; const int srcEnd = count - adjust; srcIdx = 2; for (int i = 0; i < 256; i++) map16[i] = 0x10000 | i; if (srcIdx + 3 * n > srcEnd) return false; for (int i = 0; i < n; i++) { map16[int(src[srcIdx + 2])] = 0x20000 | int(src[srcIdx]) | (int(src[srcIdx + 1]) << 8); srcIdx += 3; } const int nbAliases = srcEnd - srcIdx; const int dstAvail = dstEnd - dstIdx; if (nbAliases <= (dstAvail >> 1)) { while (srcIdx < srcEnd) { const int val = map16[int(src[srcIdx++])]; dst[dstIdx] = kanzi::byte(val); dst[dstIdx + 1] = kanzi::byte(val >> 8); dstIdx += (val >> 16); } } else { while ((srcIdx < srcEnd) && (dstIdx + 1 < dstEnd)) { const int val = map16[int(src[srcIdx++])]; dst[dstIdx] = kanzi::byte(val); dst[dstIdx + 1] = kanzi::byte(val >> 8); dstIdx += (val >> 16); } while (srcIdx < srcEnd) { const int val = map16[int(src[srcIdx++])]; const int inc = val >> 16; if (dstIdx + inc > dstEnd) return false; dst[dstIdx + inc - 1] = kanzi::byte(val >> 8); dst[dstIdx] = kanzi::byte(val); dstIdx += inc; } } if (adjust != 0) { if ((srcIdx >= count) || (dstIdx >= dstEnd)) return false; dst[dstIdx++] = src[srcIdx++]; } } input._index += srcIdx; output._index += dstIdx; return srcIdx == count; } kanzi-cpp-2.6.0/src/transform/AliasCodec.hpp000066400000000000000000000033201525576310500207240ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_AliasCodec #define knz_AliasCodec #include "../Context.hpp" #include "../Transform.hpp" namespace kanzi { typedef struct ssAlias { uint32 val; uint32 freq; ssAlias(uint32 v, uint32 f) : val(v), freq(f) { } friend bool operator< (ssAlias const& lhs, ssAlias const& rhs) { int r; return ((r = int(lhs.freq - rhs.freq)) != 0) ? r > 0 : lhs.val > rhs.val; } } sdAlias; // Simple codec replacing large symbols with small aliases whenever possible class AliasCodec FINAL : public Transform { public: AliasCodec() { _pCtx = nullptr; _onlyDNA = false; } AliasCodec(Context& ctx); ~AliasCodec() {} bool forward(SliceArray& src, SliceArray& dst, int length); bool inverse(SliceArray& src, SliceArray& dst, int length); // Required encoding output buffer size int getMaxEncodedLength(int srcLen) const { return srcLen + 1024; } private: static const int MIN_BLOCK_SIZE; Context* _pCtx; bool _onlyDNA; }; } #endif kanzi-cpp-2.6.0/src/transform/BWT.cpp000066400000000000000000000700641525576310500173750ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include "BWT.hpp" #include "../Global.hpp" #include "../Memory.hpp" #ifdef CONCURRENCY_ENABLED #include #endif using namespace kanzi; using namespace std; const int BWT::MAX_BLOCK_SIZE = 1024 * 1024 * 1024; // 1024 MB const int BWT::NB_FASTBITS = 17; const int BWT::MASK_FASTBITS = (1 << NB_FASTBITS) - 1; const int BWT::BLOCK_SIZE_THRESHOLD1 = 256; const int BWT::BLOCK_SIZE_THRESHOLD2 = 2 * 1024 * 1024; #define DECODE_BWT(P, S) \ do { \ (S) = _fastBits[(P) >> shift]; \ if (_buckets[(S)] <= (P)) { \ do { \ (S)++; \ } while (_buckets[(S)] <= (P)); \ } \ } while (0) BWT::BWT(int jobs) { _buffer = nullptr; _sa = nullptr; _bufferSize = 0; _saSize = 0; #ifdef CONCURRENCY_ENABLED _pool = nullptr; if (jobs < 1) throw invalid_argument("The number of jobs must be at least 1"); #else if (jobs != 1) throw invalid_argument("The number of jobs is limited to 1 in this version"); #endif _jobs = jobs; memset(_primaryIndexes, 0, sizeof(int) * 8); } BWT::BWT(Context& ctx) { _buffer = nullptr; _sa = nullptr; _bufferSize = 0; _saSize = 0; int jobs = ctx.getInt("jobs", 1); #ifdef CONCURRENCY_ENABLED _pool = ctx.getPool(); // can be null if (jobs < 1) throw invalid_argument("The number of jobs must be at least 1"); #else if (jobs != 1) throw invalid_argument("The number of jobs is limited to 1 in this version"); #endif _jobs = jobs; memset(_primaryIndexes, 0, sizeof(int) * 8); } bool BWT::setPrimaryIndex(int n, int primaryIndex) { if ((primaryIndex < 0) || (n < 0) || (n >= 8)) return false; _primaryIndexes[n] = primaryIndex; return true; } bool BWT::forward(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (!SliceArray::isValid(input)) throw invalid_argument("BWT: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("BWT: Invalid output block"); if ((count < 0) || (count > input._length - input._index) || (count > output._length - output._index) || (count > MAX_BLOCK_SIZE)) return false; if (count == 1) { if ((input._index >= input._length) || (output._index >= output._length)) return false; output._array[output._index++] = input._array[input._index++]; return true; } const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; // Lazy dynamic memory allocation if (_saSize < count) { int* sa = new int[count]; delete[] _sa; _sa = sa; _saSize = count; } if (_saAlgo.computeBWT(src, dst, _sa, count, _primaryIndexes, getBWTChunks(count)) == false) return false; input._index += count; output._index += count; return true; } bool BWT::inverse(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (!SliceArray::isValid(input)) throw invalid_argument("BWT: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("BWT: Invalid output block"); if ((count < 0) || (count > input._length - input._index) || (count > output._length - output._index) || (count > MAX_BLOCK_SIZE)) return false; if (count == 1) { if ((input._index >= input._length) || (output._index >= output._length)) return false; output._array[output._index++] = input._array[input._index++]; return true; } // Find the fastest way to implement inverse based on block size if (count <= BLOCK_SIZE_THRESHOLD2) return inverseMergeTPSI(input, output, count); return inverseBiPSIv2(input, output, count); } // When count <= BLOCK_SIZE_THRESHOLD2, mergeTPSI algo bool BWT::inverseMergeTPSI(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; const int pIdx = getPrimaryIndex(0); if ((pIdx <= 0) || (pIdx > count)) return false; // Lazy dynamic memory allocation if (_bufferSize < count) { const int newSize = max(count, 256); uint* buffer = new uint[newSize]; delete[] _buffer; _buffer = buffer; _bufferSize = newSize; } // Build array of packed index + value (assumes block size < 1<<24) uint buckets[256] = { 0 }; Global::computeHistogram(&input._array[input._index], count, buckets); for (int i = 0, sum = 0; i < 256; i++) { const int tmp = buckets[i]; buckets[i] = sum; sum += tmp; } const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; const uint end1 = uint(pIdx); const uint end2 = uint(count); _buffer[buckets[uint8(src[0])]] = uint(src[0]); buckets[uint8(src[0])]++; for (uint i = 1; i < end1; i++) { const uint8 val = uint8(src[i]); _buffer[buckets[val]] = ((i - 1) << 8) | val; buckets[val]++; } for (uint i = end1; i < end2; i++) { const uint8 val = uint8(src[i]); _buffer[buckets[val]] = (i << 8) | val; buckets[val]++; } if (getBWTChunks(count) != 8) { int t = pIdx - 1; int n = 0; while (n < count) { const int ptr = _buffer[t]; dst[n++] = kanzi::byte(ptr); t = ptr >> 8; } } else { const int ckSize = ((count & 7) == 0) ? count >> 3 : (count >> 3) + 1; int t0 = getPrimaryIndex(0) - 1; if ((t0 < 0) || (t0 >= count)) return false; int t1 = getPrimaryIndex(1) - 1; if ((t1 < 0) || (t1 >= count)) return false; int t2 = getPrimaryIndex(2) - 1; if ((t2 < 0) || (t2 >= count)) return false; int t3 = getPrimaryIndex(3) - 1; if ((t3 < 0) || (t3 >= count)) return false; int t4 = getPrimaryIndex(4) - 1; if ((t4 < 0) || (t4 >= count)) return false; int t5 = getPrimaryIndex(5) - 1; if ((t5 < 0) || (t5 >= count)) return false; int t6 = getPrimaryIndex(6) - 1; if ((t6 < 0) || (t6 >= count)) return false; int t7 = getPrimaryIndex(7) - 1; if ((t7 < 0) || (t7 >= count)) return false; // Last interval [7*chunk:count] smaller when 8*ckSize != count const int end = count - ckSize * 7; kanzi::byte* d0 = &dst[end + ckSize * 0]; kanzi::byte* d1 = &dst[end + ckSize * 1]; kanzi::byte* d2 = &dst[end + ckSize * 2]; kanzi::byte* d3 = &dst[end + ckSize * 3]; kanzi::byte* d4 = &dst[end + ckSize * 4]; kanzi::byte* d5 = &dst[end + ckSize * 5]; kanzi::byte* d6 = &dst[end + ckSize * 6]; kanzi::byte* d7 = &dst[end + ckSize * 7]; int n = -end; int ptr; #define S(t, d) ptr = _buffer[t]; \ d[n] = kanzi::byte(ptr); \ t = ptr >> 8 while (n < 0) { S(t0, d0); S(t1, d1); S(t2, d2); S(t3, d3); S(t4, d4); S(t5, d5); S(t6, d6); S(t7, d7); n++; } while (n < ckSize - end) { S(t0, d0); S(t1, d1); S(t2, d2); S(t3, d3); S(t4, d4); S(t5, d5); S(t6, d6); n++; } } input._index += count; output._index += count; return true; } // When count > BLOCK_SIZE_THRESHOLD2, biPSIv2 algo bool BWT::inverseBiPSIv2(SliceArray& input, SliceArray& output, int count) { // Lazy dynamic memory allocations if (_bufferSize < count + 1) { const int newSize = max(count + 1, 256); uint* buffer = new uint[newSize]; delete[] _buffer; _buffer = buffer; _bufferSize = newSize; } const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; const int pIdx = getPrimaryIndex(0); if ((pIdx <= 0) || (pIdx > count)) return false; for (int i = 1; i < 8; i++) { const int p = getPrimaryIndex(i); if ((p <= 0) || (p > count)) return false; } uint* buckets = nullptr; uint16* fastBits = nullptr; #ifdef CONCURRENCY_ENABLED vector*> tasks; #endif try { buckets = new uint[65536]; memset(&buckets[0], 0, 65536 * sizeof(uint)); uint freqs[256] = { 0 }; Global::computeHistogram(&input._array[input._index], count, freqs); for (int sum = 1, c = 0; c < 256; c++) { const int f = sum; sum += int(freqs[c]); freqs[c] = f; if (f != sum) { uint* ptr = &buckets[c << 8]; const int hi = min(sum, pIdx); for (int i = f; i < hi; i++) ptr[int(src[i])]++; const int lo = max(f - 1, pIdx); for (int i = lo; i < sum - 1; i++) ptr[int(src[i])]++; } } const int lastc = int(src[0]); fastBits = new uint16[MASK_FASTBITS + 1]; memset(&fastBits[0], 0, size_t(MASK_FASTBITS + 1) * sizeof(uint16)); int shift = 0; while ((count >> shift) > MASK_FASTBITS) shift++; for (int v = 0, sum = 1, c = 0; c < 256; c++) { if (c == lastc) sum++; uint* ptr = &buckets[c]; for (int d = 0; d < 256; d++) { const int s = sum; sum += ptr[d << 8]; ptr[d << 8] = s; if (s == sum) continue; for (; v <= ((sum - 1) >> shift); v++) fastBits[v] = uint16((c << 8) | d); } } memset(&_buffer[0], 0, size_t(count) * sizeof(uint)); int n = 0; while (n < pIdx) { const int c = int(src[n]); const int p = freqs[c]; if (p < pIdx) _buffer[buckets[(c << 8) | int(src[p])]++] = n; else if (p > pIdx) _buffer[buckets[(c << 8) | int(src[p - 1])]++] = n; freqs[c]++; n++; } while (n < count) { const int c = int(src[n]); const int p = freqs[c]; freqs[c]++; n++; if (p < pIdx) _buffer[buckets[(c << 8) | int(src[p])]++] = n; else if (p > pIdx) _buffer[buckets[(c << 8) | int(src[p - 1])]++] = n; } for (int c = 0; c < 256; c++) { for (int d = 0; d < c; d++) { swap(buckets[(d << 8) | c], buckets[(c << 8) | d]); } } const int chunks = getBWTChunks(count); // Build inverse const int st = count / chunks; const int ckSize = (chunks * st == count) ? st : st + 1; const int nbTasks = (_jobs < chunks) ? _jobs : chunks; if (nbTasks == 1) { InverseBiPSIv2Task task(_buffer, buckets, fastBits, dst, _primaryIndexes, count, 0, ckSize, 0, chunks); task.run(); } else { #ifdef CONCURRENCY_ENABLED // Several chunks may be decoded concurrently (depending on the availability // of jobs per block). int jobsPerTask[64]; Global::computeJobsPerTask(jobsPerTask, chunks, nbTasks); vector > futures; tasks.reserve(nbTasks); futures.reserve(nbTasks); try { // Create one task per job for (int j = 0, c = 0; j < nbTasks; j++) { // Each task decodes jobsPerTask[j] chunks tasks.push_back(new InverseBiPSIv2Task(_buffer, buckets, fastBits, dst, _primaryIndexes, count, c * ckSize, ckSize, c, c + jobsPerTask[j])); if (_pool == nullptr) futures.push_back(std::async(launch::async, &InverseBiPSIv2Task::run, tasks[j])); else futures.push_back(_pool->schedule(&InverseBiPSIv2Task::run, tasks[j])); c += jobsPerTask[j]; } // Wait for completion of all concurrent tasks for (int j = 0; j < nbTasks; j++) futures[j].get(); } catch (...) { for (uint i = 0; i < futures.size(); i++) { try { if (futures[i].valid()) futures[i].wait(); } catch (const exception&) { } } throw; } #else // nbTasks > 1 but concurrency is not enabled (should never happen) throw invalid_argument("Error during BWT inverse: concurrency not supported"); #endif } dst[count - 1] = kanzi::byte(lastc); input._index += count; output._index += count; } catch (...) { #ifdef CONCURRENCY_ENABLED for (uint i = 0; i < tasks.size(); i++) delete tasks[i]; #endif delete[] fastBits; delete[] buckets; throw; } #ifdef CONCURRENCY_ENABLED for (uint i = 0; i < tasks.size(); i++) delete tasks[i]; #endif delete[] fastBits; delete[] buckets; return true; } template InverseBiPSIv2Task::InverseBiPSIv2Task(uint* buf, uint* buckets, uint16* fastBits, kanzi::byte* output, int* primaryIndexes, int total, int start, int ckSize, int firstChunk, int lastChunk) : _data(buf) , _buckets(buckets) , _fastBits(fastBits) , _primaryIndexes(primaryIndexes) , _dst(output) , _total(total) , _start(start) , _ckSize(ckSize) , _firstChunk(firstChunk) , _lastChunk(lastChunk) { } template T InverseBiPSIv2Task::run() { uint sh = 0; while ((_total >> sh) > BWT::MASK_FASTBITS) sh++; const uint shift = sh; int c = _firstChunk; kanzi::byte* d0 = &_dst[0 * _ckSize]; kanzi::byte* d1 = &_dst[1 * _ckSize]; kanzi::byte* d2 = &_dst[2 * _ckSize]; kanzi::byte* d3 = &_dst[3 * _ckSize]; kanzi::byte* d4 = &_dst[4 * _ckSize]; kanzi::byte* d5 = &_dst[5 * _ckSize]; kanzi::byte* d6 = &_dst[6 * _ckSize]; kanzi::byte* d7 = &_dst[7 * _ckSize]; if (_start + 7 * _ckSize <= _total) { for (; c + 8 <= _lastChunk; c += 8) { const int end = _start + _ckSize - 1; const int end8 = min(end, _total - 7 * _ckSize - 1); uint p0 = _primaryIndexes[c + 0]; uint p1 = _primaryIndexes[c + 1]; uint p2 = _primaryIndexes[c + 2]; uint p3 = _primaryIndexes[c + 3]; uint p4 = _primaryIndexes[c + 4]; uint p5 = _primaryIndexes[c + 5]; uint p6 = _primaryIndexes[c + 6]; uint p7 = _primaryIndexes[c + 7]; for (int i = _start + 1; i <= end8; i += 2) { prefetchRead(&_data[p0]); prefetchRead(&_data[p1]); prefetchRead(&_data[p2]); prefetchRead(&_data[p3]); prefetchRead(&_data[p4]); prefetchRead(&_data[p5]); prefetchRead(&_data[p6]); prefetchRead(&_data[p7]); uint16 s0 = _fastBits[p0 >> shift]; uint16 s1 = _fastBits[p1 >> shift]; uint16 s2 = _fastBits[p2 >> shift]; uint16 s3 = _fastBits[p3 >> shift]; uint16 s4 = _fastBits[p4 >> shift]; uint16 s5 = _fastBits[p5 >> shift]; uint16 s6 = _fastBits[p6 >> shift]; uint16 s7 = _fastBits[p7 >> shift]; if (_buckets[s0] <= p0) { do { s0++; } while (_buckets[s0] <= p0); } if (_buckets[s1] <= p1) { do { s1++; } while (_buckets[s1] <= p1); } if (_buckets[s2] <= p2) { do { s2++; } while (_buckets[s2] <= p2); } if (_buckets[s3] <= p3) { do { s3++; } while (_buckets[s3] <= p3); } if (_buckets[s4] <= p4) { do { s4++; } while (_buckets[s4] <= p4); } if (_buckets[s5] <= p5) { do { s5++; } while (_buckets[s5] <= p5); } if (_buckets[s6] <= p6) { do { s6++; } while (_buckets[s6] <= p6); } if (_buckets[s7] <= p7) { do { s7++; } while (_buckets[s7] <= p7); } d0[i - 1] = kanzi::byte(s0 >> 8); d0[i] = kanzi::byte(s0); d1[i - 1] = kanzi::byte(s1 >> 8); d1[i] = kanzi::byte(s1); d2[i - 1] = kanzi::byte(s2 >> 8); d2[i] = kanzi::byte(s2); d3[i - 1] = kanzi::byte(s3 >> 8); d3[i] = kanzi::byte(s3); d4[i - 1] = kanzi::byte(s4 >> 8); d4[i] = kanzi::byte(s4); d5[i - 1] = kanzi::byte(s5 >> 8); d5[i] = kanzi::byte(s5); d6[i - 1] = kanzi::byte(s6 >> 8); d6[i] = kanzi::byte(s6); d7[i - 1] = kanzi::byte(s7 >> 8); d7[i] = kanzi::byte(s7); p0 = _data[p0]; p1 = _data[p1]; p2 = _data[p2]; p3 = _data[p3]; p4 = _data[p4]; p5 = _data[p5]; p6 = _data[p6]; p7 = _data[p7]; } // Keep the eighth chain within its logical end. If the common // extent is odd, retain the low byte for the first seven chains. const bool oddCommon = ((end8 - _start + 1) & 1) != 0; if (oddCommon) { // Keep the low byte for the first seven chains; the eighth // chain ends at end8 and only needs the high byte. uint16 s0, s1, s2, s3, s4, s5, s6, s7; DECODE_BWT(p0, s0); DECODE_BWT(p1, s1); DECODE_BWT(p2, s2); DECODE_BWT(p3, s3); DECODE_BWT(p4, s4); DECODE_BWT(p5, s5); DECODE_BWT(p6, s6); DECODE_BWT(p7, s7); d0[end8] = kanzi::byte(s0 >> 8); d1[end8] = kanzi::byte(s1 >> 8); d2[end8] = kanzi::byte(s2 >> 8); d3[end8] = kanzi::byte(s3 >> 8); d4[end8] = kanzi::byte(s4 >> 8); d5[end8] = kanzi::byte(s5 >> 8); d6[end8] = kanzi::byte(s6 >> 8); d7[end8] = kanzi::byte(s7 >> 8); if (end8 < end) { d0[end8 + 1] = kanzi::byte(s0); d1[end8 + 1] = kanzi::byte(s1); d2[end8 + 1] = kanzi::byte(s2); d3[end8 + 1] = kanzi::byte(s3); d4[end8 + 1] = kanzi::byte(s4); d5[end8 + 1] = kanzi::byte(s5); d6[end8 + 1] = kanzi::byte(s6); } p0 = _data[p0]; p1 = _data[p1]; p2 = _data[p2]; p3 = _data[p3]; p4 = _data[p4]; p5 = _data[p5]; p6 = _data[p6]; p7 = _data[p7]; } // The last chunk can be shorter than the other seven. Finish the // common extent of the first seven chains without a per-iteration // boundary check for the eighth chain. if (end8 < end) { const int nextPos = end8 + (oddCommon ? 2 : 1); const int tailStart = nextPos + 1; for (int i = tailStart; i <= end; i += 2) { prefetchRead(&_data[p0]); prefetchRead(&_data[p1]); prefetchRead(&_data[p2]); prefetchRead(&_data[p3]); prefetchRead(&_data[p4]); prefetchRead(&_data[p5]); prefetchRead(&_data[p6]); uint16 s0, s1, s2, s3, s4, s5, s6; DECODE_BWT(p0, s0); DECODE_BWT(p1, s1); DECODE_BWT(p2, s2); DECODE_BWT(p3, s3); DECODE_BWT(p4, s4); DECODE_BWT(p5, s5); DECODE_BWT(p6, s6); d0[i - 1] = kanzi::byte(s0 >> 8); d0[i] = kanzi::byte(s0); d1[i - 1] = kanzi::byte(s1 >> 8); d1[i] = kanzi::byte(s1); d2[i - 1] = kanzi::byte(s2 >> 8); d2[i] = kanzi::byte(s2); d3[i - 1] = kanzi::byte(s3 >> 8); d3[i] = kanzi::byte(s3); d4[i - 1] = kanzi::byte(s4 >> 8); d4[i] = kanzi::byte(s4); d5[i - 1] = kanzi::byte(s5 >> 8); d5[i] = kanzi::byte(s5); d6[i - 1] = kanzi::byte(s6 >> 8); d6[i] = kanzi::byte(s6); p0 = _data[p0]; p1 = _data[p1]; p2 = _data[p2]; p3 = _data[p3]; p4 = _data[p4]; p5 = _data[p5]; p6 = _data[p6]; } if ((nextPos <= end) && (((end - nextPos + 1) & 1) != 0)) { prefetchRead(&_data[p0]); prefetchRead(&_data[p1]); prefetchRead(&_data[p2]); prefetchRead(&_data[p3]); prefetchRead(&_data[p4]); prefetchRead(&_data[p5]); prefetchRead(&_data[p6]); uint16 s0, s1, s2, s3, s4, s5, s6; DECODE_BWT(p0, s0); DECODE_BWT(p1, s1); DECODE_BWT(p2, s2); DECODE_BWT(p3, s3); DECODE_BWT(p4, s4); DECODE_BWT(p5, s5); DECODE_BWT(p6, s6); d0[end] = kanzi::byte(s0 >> 8); d1[end] = kanzi::byte(s1 >> 8); d2[end] = kanzi::byte(s2 >> 8); d3[end] = kanzi::byte(s3 >> 8); d4[end] = kanzi::byte(s4 >> 8); d5[end] = kanzi::byte(s5 >> 8); d6[end] = kanzi::byte(s6 >> 8); } } _start += (8 * _ckSize); } } // Preserve interleaving when a worker owns four or two chunks. if ((_start + 3 * _ckSize <= _total) && ((_ckSize & 1) == 0)) { for (; c + 4 <= _lastChunk; c += 4) { const int end = _start + _ckSize - 1; const int end4 = min(end, _total - 3 * _ckSize - 1); uint p0 = _primaryIndexes[c + 0]; uint p1 = _primaryIndexes[c + 1]; uint p2 = _primaryIndexes[c + 2]; uint p3 = _primaryIndexes[c + 3]; for (int i = _start + 1; i <= end4; i += 2) { prefetchRead(&_data[p0]); prefetchRead(&_data[p1]); prefetchRead(&_data[p2]); prefetchRead(&_data[p3]); uint16 s0, s1, s2, s3; DECODE_BWT(p0, s0); DECODE_BWT(p1, s1); DECODE_BWT(p2, s2); DECODE_BWT(p3, s3); d0[i - 1] = kanzi::byte(s0 >> 8); d0[i] = kanzi::byte(s0); d1[i - 1] = kanzi::byte(s1 >> 8); d1[i] = kanzi::byte(s1); d2[i - 1] = kanzi::byte(s2 >> 8); d2[i] = kanzi::byte(s2); d3[i - 1] = kanzi::byte(s3 >> 8); d3[i] = kanzi::byte(s3); p0 = _data[p0]; p1 = _data[p1]; p2 = _data[p2]; p3 = _data[p3]; } if (end4 < end) { const int tailStart = end4 + 1 + (end4 & 1); for (int i = tailStart; i <= end; i += 2) { prefetchRead(&_data[p0]); prefetchRead(&_data[p1]); prefetchRead(&_data[p2]); uint16 s0, s1, s2; DECODE_BWT(p0, s0); DECODE_BWT(p1, s1); DECODE_BWT(p2, s2); d0[i - 1] = kanzi::byte(s0 >> 8); d0[i] = kanzi::byte(s0); d1[i - 1] = kanzi::byte(s1 >> 8); d1[i] = kanzi::byte(s1); d2[i - 1] = kanzi::byte(s2 >> 8); d2[i] = kanzi::byte(s2); p0 = _data[p0]; p1 = _data[p1]; p2 = _data[p2]; } } _start += (4 * _ckSize); } } if ((_start + _ckSize <= _total) && ((_ckSize & 1) == 0)) { for (; c + 2 <= _lastChunk; c += 2) { const int end = _start + _ckSize - 1; const int end2 = min(end, _total - _ckSize - 1); uint p0 = _primaryIndexes[c + 0]; uint p1 = _primaryIndexes[c + 1]; for (int i = _start + 1; i <= end2; i += 2) { prefetchRead(&_data[p0]); prefetchRead(&_data[p1]); uint16 s0, s1; DECODE_BWT(p0, s0); DECODE_BWT(p1, s1); d0[i - 1] = kanzi::byte(s0 >> 8); d0[i] = kanzi::byte(s0); d1[i - 1] = kanzi::byte(s1 >> 8); d1[i] = kanzi::byte(s1); p0 = _data[p0]; p1 = _data[p1]; } if (end2 < end) { const int tailStart = end2 + 1 + (end2 & 1); for (int i = tailStart; i <= end; i += 2) { prefetchRead(&_data[p0]); uint16 s0; DECODE_BWT(p0, s0); d0[i - 1] = kanzi::byte(s0 >> 8); d0[i] = kanzi::byte(s0); p0 = _data[p0]; } } _start += (2 * _ckSize); } } for (; c < _lastChunk; c++) { const int end = min(_start + _ckSize, _total - 1); uint p = _primaryIndexes[c]; for (int i = _start + 1; i <= end; i += 2) { uint16 s = _fastBits[p >> shift]; while (_buckets[s] <= p) s++; _dst[i - 1] = kanzi::byte(s >> 8); _dst[i] = kanzi::byte(s); p = _data[p]; } _start = end; } return T(0); } #undef DECODE_BWT kanzi-cpp-2.6.0/src/transform/BWT.hpp000066400000000000000000000103101525576310500173660ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_BWT #define knz_BWT #include "../concurrent.hpp" #include "../Context.hpp" #include "../Transform.hpp" #include "DivSufSort.hpp" namespace kanzi { // The Burrows-Wheeler Transform is a reversible transform based on // permutation of the data in the original message to reduce the entropy. // The initial text can be found here: // Burrows M and Wheeler D, [A block sorting lossless data compression algorithm] // Technical Report 124, Digital Equipment Corporation, 1994 // See also Peter Fenwick, [Block sorting text compression - final report] // Technical Report 130, 1996 // This implementation replaces the 'slow' sorting of permutation strings // with the construction of a suffix array (faster but more complex). // // E.G. 0123456789A // Source: mississippi\0 // Suffixes: rank sorted // mississippi\0 0 -> 4 i\0 // ississippi\0 1 -> 3 ippi\0 // ssissippi\0 2 -> 10 issippi\0 // sissippi\0 3 -> 8 ississippi\0 // issippi\0 4 -> 2 mississippi\0 // ssippi\0 5 -> 9 pi\0 // sippi\0 6 -> 7 ppi\0 // ippi\0 7 -> 1 sippi\0 // ppi\0 8 -> 6 sissippi\0 // pi\0 9 -> 5 ssippi\0 // i\0 10 -> 0 ssissippi\0 // Suffix array SA : 10 7 4 1 0 9 8 6 3 5 2 // BWT[i] = input[SA[i]-1] => BWT(input) = ipssmpissii (+ primary index 5) // The suffix array and permutation vector are equal when the input is 0 terminated // The insertion of a guard is done internally and is entirely transparent. // // This implementation extends the canonical algorithm to use up to MAX_CHUNKS primary // indexes (based on input block size). Each primary index corresponds to a data chunk. // Chunks may be inverted concurrently. template class InverseBiPSIv2Task FINAL : public Task { private: uint* _data; uint* _buckets; uint16* _fastBits; int* _primaryIndexes; byte* _dst; int _total; int _start; int _ckSize; int _firstChunk; int _lastChunk; public: InverseBiPSIv2Task(uint* buf, uint* buckets, uint16* fastBits, byte* output, int* primaryIndexes, int total, int start, int ckSize, int firstChunk, int lastChunk); ~InverseBiPSIv2Task() {} T run(); }; class BWT FINAL : public Transform { private: static const int MAX_BLOCK_SIZE; static const int NB_FASTBITS; static const int BLOCK_SIZE_THRESHOLD1; static const int BLOCK_SIZE_THRESHOLD2; uint* _buffer; int* _sa; int _bufferSize; int _saSize; int _primaryIndexes[8]; DivSufSort _saAlgo; int _jobs; #ifdef CONCURRENCY_ENABLED ThreadPool* _pool; #endif bool inverseBiPSIv2(SliceArray& input, SliceArray& output, int count); bool inverseMergeTPSI(SliceArray& input, SliceArray& output, int count); public: static const int MASK_FASTBITS; BWT(int jobs = 1); BWT(Context& ctx); ~BWT() { if (_buffer != nullptr) delete[] _buffer; if (_sa != nullptr) delete[] _sa; } bool forward(SliceArray& input, SliceArray& output, int length); bool inverse(SliceArray& input, SliceArray& output, int length); int getPrimaryIndex(int n) const { return _primaryIndexes[n]; } bool setPrimaryIndex(int n, int primaryIndex); int getMaxEncodedLength(int srcLen) const { return srcLen; } static int getBWTChunks(int size); }; inline int BWT::getBWTChunks(int size) { return (size < BLOCK_SIZE_THRESHOLD1) ? 1 : 8; } } #endif kanzi-cpp-2.6.0/src/transform/BWTBlockCodec.cpp000066400000000000000000000124131525576310500213000ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include "BWTBlockCodec.hpp" #include "../Global.hpp" using namespace kanzi; BWTBlockCodec::BWTBlockCodec(Context& ctx) { _pBWT = new BWT(ctx); _bsVersion = ctx.getInt("bsVersion"); } // Return true if the compression chain succeeded. In this case, the input data // may be modified. If the compression failed, the input data is returned unmodified. bool BWTBlockCodec::forward(SliceArray& input, SliceArray& output, int blockSize) { if (blockSize == 0) return true; if (!SliceArray::isValid(input)) throw std::invalid_argument("BWTBlockCodec: Invalid input block"); if (!SliceArray::isValid(output)) throw std::invalid_argument("BWTBlockCodec: Invalid output block"); if (input._array == output._array) return false; if (output._length - output._index < getMaxEncodedLength(blockSize)) return false; int logBlockSize = Global::_log2(uint32(blockSize)); if ((blockSize & (blockSize - 1)) != 0) logBlockSize++; const int pIndexSize = (logBlockSize + 7) >> 3; if ((pIndexSize <= 0) || (pIndexSize >= 5)) return false; const int chunks = BWT::getBWTChunks(blockSize); const int logNbChunks = Global::_log2(uint32(chunks)); if (logNbChunks > 7) return false; kanzi::byte* dst = &output._array[output._index]; output._index += (1 + chunks * pIndexSize); // Apply forward transform if (_pBWT->forward(input, output, blockSize) == false) return false; const kanzi::byte mode = kanzi::byte((logNbChunks << 2) | (pIndexSize - 1)); // Emit header for (int i = 0, idx = 1; i < chunks; i++) { const int primaryIndex = _pBWT->getPrimaryIndex(i) - 1; int shift = (pIndexSize - 1) << 3; while (shift >= 0) { dst[idx++] = kanzi::byte(primaryIndex >> shift); shift -= 8; } } dst[0] = mode; return true; } bool BWTBlockCodec::inverse(SliceArray& input, SliceArray& output, int blockSize) { if (blockSize <= 1) return blockSize == 0; if (!SliceArray::isValid(input)) throw std::invalid_argument("BWTBlockCodec: Invalid input block"); if (!SliceArray::isValid(output)) throw std::invalid_argument("BWTBlockCodec: Invalid output block"); if (input._array == output._array) return false; if ((blockSize < 1) || (input._length - input._index < 1)) return false; if (_bsVersion > 5) { // Number of chunks and primary index size in bitstream since bsVersion 6 kanzi::byte mode = input._array[input._index++]; const uint logNbChunks = uint(mode >> 2) & 0x07; const int pIndexSize = (int(mode) & 0x03) + 1; const int chunks = 1 << logNbChunks; const int headerSize = 1 + chunks * pIndexSize; if ((input._length - input._index < headerSize) || (blockSize < headerSize)) return false; if (chunks != BWT::getBWTChunks(blockSize - headerSize)) return false; // Read header for (int i = 0; i < chunks; i++) { int shift = (pIndexSize - 1) << 3; uint primaryIndex = 0; // Extract BWT primary index while (shift >= 0) { primaryIndex = (primaryIndex << 8) | int(input._array[input._index++]); shift -= 8; } if (primaryIndex >= 0x7FFFFFFFU) return false; if (_pBWT->setPrimaryIndex(i, int(primaryIndex) + 1) == false) return false; } blockSize -= headerSize; } else { const int chunks = BWT::getBWTChunks(blockSize); for (int i = 0; i < chunks; i++) { // Read block header (mode + primary index) if (input._index >= input._length) return false; const int blockMode = int(input._array[input._index++]); const int pIndexSizeBytes = 1 + ((blockMode >> 6) & 0x03); if ((blockSize < pIndexSizeBytes) || (input._index + (pIndexSizeBytes - 1) > input._length)) return false; blockSize -= pIndexSizeBytes; int shift = (pIndexSizeBytes - 1) << 3; int primaryIndex = (blockMode & 0x3F) << shift; // Extract BWT primary index for (int n = 1; n < pIndexSizeBytes; n++) { shift -= 8; primaryIndex |= (int(input._array[input._index++]) << shift); } if (_pBWT->setPrimaryIndex(i, primaryIndex) == false) return false; } } // Apply inverse Transform return _pBWT->inverse(input, output, blockSize); } kanzi-cpp-2.6.0/src/transform/BWTBlockCodec.hpp000066400000000000000000000031321525576310500213030ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_BWTBlockCodec #define knz_BWTBlockCodec #include "../transform/BWT.hpp" #include "../Context.hpp" namespace kanzi { // Utility class to en/de-code a BWT data block and its associated primary index(es) // BWT stream format: Header (mode + primary index(es)) | Data (n bytes) // mode (8 bits): xxxyyyzz // xxx: ignored // yyy: log(chunks) // zz: primary index size - 1 (in bytes) // primary indexes (chunks * (8|16|24|32 bits)) class BWTBlockCodec FINAL : public Transform { public: BWTBlockCodec(Context& ctx); ~BWTBlockCodec() { delete _pBWT; } bool forward(SliceArray& input, SliceArray& output, int length); bool inverse(SliceArray& input, SliceArray& output, int length); // Required encoding output buffer size int getMaxEncodedLength(int srcLen) const { return srcLen + 1 + 32; // mode + 8 indexes } private: BWT* _pBWT; int _bsVersion; }; } #endif kanzi-cpp-2.6.0/src/transform/BWTS.cpp000066400000000000000000000153731525576310500175220ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include "BWTS.hpp" #include "../Global.hpp" using namespace kanzi; using namespace std; const int BWTS::MAX_BLOCK_SIZE = 1024 * 1024 * 1024; // 1024 MB bool BWTS::forward(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (!SliceArray::isValid(input)) throw invalid_argument("BWTS: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("BWTS: Invalid output block"); if ((count < 0) || (count > input._length - input._index) || (count > output._length - output._index)) return false; if (count > MAX_BLOCK_SIZE) { // Not a recoverable error: instead of silently fail the transform, // issue a fatal error. stringstream ss; ss << "The max BWTS block size is " << MAX_BLOCK_SIZE << ", got " << count; throw invalid_argument(ss.str()); } if (count < 2) { if (count == 1) { if ((input._index >= input._length) || (output._index >= output._length)) return false; output._array[output._index++] = input._array[input._index++]; } return true; } const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; // Lazy dynamic memory allocation if (_bufferSize < count) { int* buffer1 = new int[count]; int* buffer2 = nullptr; try { buffer2 = new int[count]; } catch (...) { delete[] buffer1; throw; } delete[] _buffer1; delete[] _buffer2; _buffer1 = buffer1; _buffer2 = buffer2; _bufferSize = count; } // Aliasing int* sa = _buffer1; int* isa = _buffer2; if (_saAlgo.computeSuffixArray(src, sa, count) == false) return false; for (int i = 0; i < count; i++) isa[sa[i]] = i; int min = isa[0]; int idxMin = 0; for (int i = 1; ((i < count) && (min > 0)); i++) { if (isa[i] >= min) continue; int refRank = moveLyndonWordHead(sa, isa, src, count, idxMin, i - idxMin, min); for (int j = i - 1; j > idxMin; j--) { // Iterate through the new Lyndon word from end to start int testRank = isa[j]; int startRank = testRank; while (testRank < count - 1) { int nextRankStart = sa[testRank + 1]; if ((j > nextRankStart) || (src[j] != src[nextRankStart]) || (refRank < isa[nextRankStart + 1])) break; sa[testRank] = nextRankStart; isa[nextRankStart] = testRank; testRank++; } sa[testRank] = j; isa[j] = testRank; refRank = testRank; if (startRank == testRank) break; } min = isa[i]; idxMin = i; } min = count; for (int i = 0; i < count; i++) { if (isa[i] >= min) { dst[isa[i]] = src[i - 1]; continue; } if (min < count) dst[min] = src[i - 1]; min = isa[i]; } dst[0] = src[count - 1]; input._index += count; output._index += count; return true; } int BWTS::moveLyndonWordHead(int sa[], int isa[], const kanzi::byte data[], int count, int start, int size, int rank) const { const int end = start + size; while (rank + 1 < count) { const int nextStart0 = sa[rank + 1]; if (nextStart0 <= end) break; int nextStart = nextStart0; int k = 0; while ((k < size) && (nextStart < count) && (data[start + k] == data[nextStart])) { k++; nextStart++; } if ((k == size) && (rank < isa[nextStart])) break; if ((k < size) && (nextStart < count) && (data[start + k] < data[nextStart])) break; sa[rank] = nextStart0; isa[nextStart0] = rank; rank++; } sa[rank] = start; isa[start] = rank; return rank; } bool BWTS::inverse(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (!SliceArray::isValid(input)) throw invalid_argument("BWTS: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("BWTS: Invalid output block"); if ((count < 0) || (count > input._length - input._index) || (count > output._length - output._index)) return false; if (count > MAX_BLOCK_SIZE) { stringstream ss; ss << "The max BWTS block size is " << MAX_BLOCK_SIZE << ", got " << count; throw invalid_argument(ss.str()); } if (count < 2) { if (count == 1) { if ((input._index >= input._length) || (output._index >= output._length)) return false; output._array[output._index++] = input._array[input._index++]; } return true; } // Lazy dynamic memory allocation if (_bufferSize < count) { int* buffer1 = new int[count]; delete[] _buffer1; _buffer1 = buffer1; _bufferSize = count; } // Initialize histogram uint buckets[256] = { 0 }; Global::computeHistogram(&input._array[input._index], count, buckets, true); // Histogram for (int i = 0, sum = 0; i < 256; i++) { sum += buckets[i]; buckets[i] = sum - buckets[i]; } // Aliasing int* lf = _buffer1; const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; for (int i = 0; i < count; i++) lf[i] = buckets[int(src[i])]++; // Build inverse for (int i = 0, j = count - 1; j >= 0; i++) { if (lf[i] < 0) continue; int p = i; do { dst[j] = src[p]; j--; const int t = lf[p]; lf[p] = -1; p = t; } while (lf[p] >= 0); } input._index += count; output._index += count; return true; } kanzi-cpp-2.6.0/src/transform/BWTS.hpp000066400000000000000000000040711525576310500175200ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_BWTS #define knz_BWTS #include "../Context.hpp" #include "../Transform.hpp" #include "DivSufSort.hpp" namespace kanzi { // Bijective version of the Burrows-Wheeler Transform // The main advantage over the regular BWT is that there is no need for a primary // index (hence the bijectivity). BWTS is about 10% slower than BWT. // Forward transform based on the code at https://code.google.com/p/mk-bwts/ // by Neal Burns and DivSufSort (port of libDivSufSort by Yuta Mori) class BWTS FINAL : public Transform { private: static const int MAX_BLOCK_SIZE; int* _buffer1; int* _buffer2; int _bufferSize; DivSufSort _saAlgo; int moveLyndonWordHead(int sa[], int isa[], const byte data[], int count, int start, int size, int rank) const; public: BWTS() { _buffer1 = nullptr; _buffer2 = nullptr; _bufferSize = 0; } BWTS(Context&) { _buffer1 = nullptr; _buffer2 = nullptr; _bufferSize = 0; } ~BWTS() { if (_buffer1 != nullptr) delete[] _buffer1; if (_buffer2 != nullptr) delete[] _buffer2; } bool forward(SliceArray& input, SliceArray& output, int length); bool inverse(SliceArray& input, SliceArray& output, int length); int getMaxEncodedLength(int srcLen) const { return srcLen; } }; } #endif kanzi-cpp-2.6.0/src/transform/DivSufSort.cpp000066400000000000000000001632121525576310500210070ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include "DivSufSort.hpp" #include "../Memory.hpp" using namespace kanzi; const int DivSufSort::SS_INSERTIONSORT_THRESHOLD = 16; const int DivSufSort::SS_BLOCKSIZE = 8192; const int DivSufSort::SS_MISORT_STACKSIZE = 16; const int DivSufSort::SS_SMERGE_STACKSIZE = 32; const int DivSufSort::TR_STACKSIZE = 64; const int DivSufSort::TR_INSERTIONSORT_THRESHOLD = 16; const int DivSufSort::SQQ_TABLE[] = { 0, 16, 22, 27, 32, 35, 39, 42, 45, 48, 50, 53, 55, 57, 59, 61, 64, 65, 67, 69, 71, 73, 75, 76, 78, 80, 81, 83, 84, 86, 87, 89, 90, 91, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 106, 107, 108, 109, 110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 128, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 144, 145, 146, 147, 148, 149, 150, 150, 151, 152, 153, 154, 155, 155, 156, 157, 158, 159, 160, 160, 161, 162, 163, 163, 164, 165, 166, 167, 167, 168, 169, 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, 178, 179, 180, 181, 181, 182, 183, 183, 184, 185, 185, 186, 187, 187, 188, 189, 189, 190, 191, 192, 192, 193, 193, 194, 195, 195, 196, 197, 197, 198, 199, 199, 200, 201, 201, 202, 203, 203, 204, 204, 205, 206, 206, 207, 208, 208, 209, 209, 210, 211, 211, 212, 212, 213, 214, 214, 215, 215, 216, 217, 217, 218, 218, 219, 219, 220, 221, 221, 222, 222, 223, 224, 224, 225, 225, 226, 226, 227, 227, 228, 229, 229, 230, 230, 231, 231, 232, 232, 233, 234, 234, 235, 235, 236, 236, 237, 237, 238, 238, 239, 240, 240, 241, 241, 242, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247, 247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255 }; const int DivSufSort::LOG_TABLE[] = { -1, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7 }; DivSufSort::DivSufSort() { _ssStack = new Stack(SS_MISORT_STACKSIZE); _trStack = new Stack(TR_STACKSIZE); _mergeStack = new Stack(SS_SMERGE_STACKSIZE); _sa = nullptr; _buffer = nullptr; _ssStack->_index = 0; _trStack->_index = 0; _mergeStack->_index = 0; memset(&_bucketA[0], 0, sizeof(int) * 256); memset(&_bucketB[0], 0, sizeof(int) * 65536); } DivSufSort::~DivSufSort() { delete _ssStack; delete _trStack; delete _mergeStack; } void DivSufSort::reset() { _ssStack->_index = 0; _trStack->_index = 0; _mergeStack->_index = 0; memset(&_bucketA[0], 0, sizeof(int) * 256); memset(&_bucketB[0], 0, sizeof(int) * 65536); } bool DivSufSort::computeSuffixArray(const kanzi::byte input[], int sa[], int length) { _buffer = reinterpret_cast(&input[0]); _sa = sa; reset(); const int m = sortTypeBstar(_bucketA, _bucketB, length); if (m < 0) return false; constructSuffixArray(_bucketA, _bucketB, length, m); return true; } void DivSufSort::constructSuffixArray(int bucketA[], int bucketB[], int n, int m) { if (m > 0) { for (int c1 = 254; c1 >= 0; c1--) { const int idx = c1 << 8; const int i = bucketB[idx + c1 + 1]; int k = 0; int c2 = -1; for (int j = bucketA[c1 + 1] - 1; j >= i; j--) { int s = _sa[j]; _sa[j] = ~s; if (s <= 0) continue; s--; const int c0 = _buffer[s]; if ((s > 0) && (_buffer[s - 1] > c0)) s = ~s; if (c0 != c2) { if (c2 >= 0) bucketB[idx + c2] = k; c2 = c0; k = bucketB[idx + c2]; } _sa[k--] = s; } } } int c2 = _buffer[n - 1]; int k = bucketA[c2]; _sa[k++] = (_buffer[n - 2] < c2) ? ~(n - 1) : (n - 1); // Scan the suffix array from left to right. for (int i = 0; i < n; i++) { int s = _sa[i]; if (s <= 0) { _sa[i] = ~s; continue; } s--; const int c0 = _buffer[s]; if ((s == 0) || (_buffer[s - 1] < c0)) s = ~s; if (c0 != c2) { bucketA[c2] = k; c2 = c0; k = bucketA[c2]; } _sa[k++] = s; } } bool DivSufSort::computeBWT(const kanzi::byte input[], kanzi::byte output[], int bwt[], int length, int indexes[], int idxCount) { _buffer = reinterpret_cast(&input[0]); _sa = bwt; reset(); const int m = sortTypeBstar(_bucketA, _bucketB, length); if (m < 0) return false; const int pIdx = constructBWT(_bucketA, _bucketB, length, m, indexes, idxCount); if (pIdx < 0) return false; output[0] = input[length - 1]; for (int i = 0; i < pIdx; i++) output[i + 1] = kanzi::byte(bwt[i]); for (int i = pIdx + 1; i < length; i++) output[i] = kanzi::byte(bwt[i]); return true; } int DivSufSort::constructBWT(int bucketA[], int bucketB[], int n, int m, int indexes[], int idxCount) { int pIdx = -1; const int st = n / idxCount; const int step = (idxCount * st == n) ? st : st + 1; if (m > 0) { for (int c1 = 254; c1 >= 0; c1--) { const int idx = c1 << 8; const int i = bucketB[idx + c1 + 1]; int k = 0; int c2 = -1; for (int j = bucketA[c1 + 1] - 1; j >= i; j--) { int s = _sa[j]; if (s <= 0) { if (s != 0) _sa[j] = ~s; continue; } if ((s % step) == 0) indexes[s / step] = j + 1; s--; const int c0 = _buffer[s]; _sa[j] = ~c0; if ((s > 0) && (_buffer[s - 1] > c0)) s = ~s; if (c0 != c2) { if (c2 >= 0) bucketB[idx + c2] = k; c2 = c0; k = bucketB[idx + c2]; } _sa[k--] = s; } } } int c2 = _buffer[n - 1]; int k = bucketA[c2]; if (_buffer[n - 2] < c2) { if (((n - 1) % step) == 0) indexes[(n - 1) / step] = n; _sa[k++] = ~_buffer[n - 2]; } else { _sa[k++] = n - 1; } // Scan the suffix array from left to right. for (int i = 0; i < n; i++) { int s = _sa[i]; if (s <= 0) { if (s != 0) _sa[i] = ~s; else pIdx = i; continue; } if ((s % step) == 0) indexes[s / step] = i + 1; s--; const int c0 = _buffer[s]; _sa[i] = c0; if (c0 != c2) { bucketA[c2] = k; c2 = c0; k = bucketA[c2]; } if ((s > 0) && (_buffer[s - 1] < c0)) { if ((s % step) == 0) { indexes[s / step] = k + 1; } s = ~_buffer[s - 1]; } _sa[k++] = s; } indexes[0] = pIdx + 1; return pIdx; } int DivSufSort::sortTypeBstar(int bucketA[], int bucketB[], int n) { int m = n; int c0 = _buffer[n - 1]; // Count the number of occurrences of the first one or two characters of each // type A, B and B* suffix. Moreover, store the beginning position of all // type B* suffixes into the array _sa. for (int i = n - 1; i >= 0;) { int c1; do { c1 = c0; bucketA[c1]++; i--; if (i < 0) break; } while ((c0 = _buffer[i]) >= c1); if (i < 0) break; bucketB[(c0 << 8) + c1]++; m--; _sa[m] = i; i--; c1 = c0; while (i >= 0) { if ((c0 = _buffer[i]) > c1) break; bucketB[(c1 << 8) + c0]++; c1 = c0; i--; } } m = n - m; c0 = 0; // A type B* suffix is lexicographically smaller than a type B suffix that // begins with the same first two characters. // Calculate the index of start/end point of each bucket. for (int i = 0, j = 0; c0 < 256; c0++) { const int t = i + bucketA[c0]; bucketA[c0] = i + j; // start point const int idx = c0 << 8; i = t + bucketB[idx + c0]; for (int c1 = c0 + 1; c1 < 256; c1++) { j += bucketB[idx + c1]; bucketB[idx + c1] = j; // end point i += bucketB[(c1 << 8) + c0]; } } if (m > 0) { // Sort the type B* suffixes by their first two characters. const int pab = n - m; for (int i = m - 2; i >= 0; i--) { const int t = _sa[pab + i]; const int idx = (_buffer[t] << 8) + _buffer[t + 1]; bucketB[idx]--; _sa[bucketB[idx]] = i; } const int t = _sa[pab + m - 1]; c0 = (_buffer[t] << 8) + _buffer[t + 1]; bucketB[c0]--; _sa[bucketB[c0]] = m - 1; // Sort the type B* substrings using ssSort. const int bufSize = n - m - m; c0 = 254; for (int j = m; j > 0; c0--) { const int idx = c0 << 8; for (int c1 = 255; c1 > c0; c1--) { const int i = bucketB[idx + c1]; if (j > i + 1) ssSort(pab, i, j, m, bufSize, 2, n, _sa[i] == m - 1); j = i; } } // Compute ranks of type B* substrings. for (int i = m - 1; i >= 0; i--) { if (_sa[i] >= 0) { const int j = i; do { _sa[m + _sa[i]] = i; i--; } while ((i >= 0) && (_sa[i] >= 0)); _sa[i + 1] = i - j; if (i <= 0) break; } const int j = i; do { _sa[i] = ~_sa[i]; _sa[m + _sa[i]] = j; i--; } while (_sa[i] < 0); _sa[m + _sa[i]] = j; } // Construct the inverse suffix array of type B* suffixes using trSort. trSort(m, 1); // Set the sorted order of type B* suffixes. c0 = _buffer[n - 1]; for (int i = n - 1, j = m; i >= 0;) { i--; for (int c1 = c0; i >= 0; i--) { if ((c0 = _buffer[i]) < c1) break; c1 = c0; } if (i >= 0) { const int tt = i; i--; for (int c1 = c0; i >= 0; i--) { if ((c0 = _buffer[i]) > c1) break; c1 = c0; } j--; _sa[_sa[m + j]] = ((tt == 0) || (tt - i > 1)) ? tt : ~tt; } } // Calculate the index of start/end point of each bucket. bucketB[65535] = n; // end int k = m - 1; for (c0 = 254; c0 >= 0; c0--) { int i = bucketA[c0 + 1] - 1; const int idx = c0 << 8; for (int c1 = 255; c1 > c0; c1--) { const int tt = i - bucketB[(c1 << 8) + c0]; bucketB[(c1 << 8) + c0] = i; // end point i = tt; const int j = bucketB[idx + c1]; // Move all type B* suffixes to the correct position. for (; k >= j; i--, k--) _sa[i] = _sa[k]; } bucketB[idx + c0 + 1] = i - bucketB[idx + c0] + 1; // start point bucketB[idx + c0] = i; // end point } } return m; } // Sub String Sort void DivSufSort::ssSort(const int pa, int first, int last, int buf, int bufSize, int depth, int n, bool lastSuffix) { if (lastSuffix == true) first++; int limit = 0; int middle = last; if ((bufSize < SS_BLOCKSIZE) && (bufSize < last - first)) { limit = ssIsqrt(last - first); if (bufSize < limit) { limit = limit > SS_BLOCKSIZE ? SS_BLOCKSIZE : limit; middle = last - limit; buf = middle; bufSize = limit; } else { limit = 0; } } int a; int i = 0; for (a = first; middle - a > SS_BLOCKSIZE; a += SS_BLOCKSIZE, i++) { ssMultiKeyIntroSort(pa, a, a + SS_BLOCKSIZE, depth); int curBufSize = last - (a + SS_BLOCKSIZE); int curBuf; if (curBufSize > bufSize) { curBuf = a + SS_BLOCKSIZE; } else { curBufSize = bufSize; curBuf = buf; } int k = SS_BLOCKSIZE; int b = a; for (int j = i; (j & 1) != 0; j >>= 1) { ssSwapMerge(pa, b - k, b, b + k, curBuf, curBufSize, depth); b -= k; k <<= 1; } } ssMultiKeyIntroSort(pa, a, middle, depth); for (int k = SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) { if ((i & 1) == 0) continue; ssSwapMerge(pa, a - k, a, middle, buf, bufSize, depth); a -= k; } if (limit != 0) { ssMultiKeyIntroSort(pa, middle, last, depth); ssInplaceMerge(pa, first, middle, last, depth); } if (lastSuffix == true) { i = _sa[first - 1]; const int p1 = _sa[pa + i]; const int p11 = n - 2; for (a = first; (a < last) && ((_sa[a] < 0) || (ssCompare(p1, p11, pa + _sa[a], depth) > 0)); a++) _sa[a - 1] = _sa[a]; _sa[a - 1] = i; } } int DivSufSort::ssCompare(int pa, int pb, int p2, const int depth) const { int u1 = depth + pa; int u2 = depth + _sa[p2]; const int u1n = pb + 2; const int u2n = _sa[p2 + 1] + 2; if (u1n - u1 > u2n - u2) { while ((u2 < u2n) && (_buffer[u1] == _buffer[u2])) { u1++; u2++; } } else { while ((u1 < u1n) && (_buffer[u1] == _buffer[u2])) { u1++; u2++; } } return (u1 < u1n) ? ((u2 < u2n) ? _buffer[u1] - _buffer[u2] : 1) : ((u2 < u2n) ? -1 : 0); } int DivSufSort::ssCompare(const int sa1[], const int sa2[], const int depth) const { int u1 = depth + sa1[0]; int u2 = depth + sa2[0]; const int u1n = sa1[1] + 2; const int u2n = sa2[1] + 2; if (u1n - u1 > u2n - u2) { while ((u2 < u2n) && (_buffer[u1] == _buffer[u2])) { u1++; u2++; } } else { while ((u1 < u1n) && (_buffer[u1] == _buffer[u2])) { u1++; u2++; } } return (u1 < u1n) ? ((u2 < u2n) ? _buffer[u1] - _buffer[u2] : 1) : ((u2 < u2n) ? -1 : 0); } void DivSufSort::ssInplaceMerge(int pa, int first, int middle, int last, int depth) { while (true) { int p, x; if (_sa[last - 1] < 0) { x = 1; p = pa + ~_sa[last - 1]; } else { x = 0; p = pa + _sa[last - 1]; } int a = first; int r = -1; for (int len = middle - first, half = (len >> 1); len > 0; len = half, half >>= 1) { const int b = a + half; const int q = ssCompare(&_sa[pa + ((_sa[b] >= 0) ? _sa[b] : ~_sa[b])], &_sa[p], depth); if (q < 0) { a = b + 1; half -= ((len & 1) ^ 1); } else r = q; } if (a < middle) { if (r == 0) _sa[a] = ~_sa[a]; ssRotate(a, middle, last); last -= (middle - a); middle = a; if (first == middle) break; } last--; if (x != 0) { last--; while (_sa[last] < 0) last--; } if (middle == last) break; } } void DivSufSort::ssRotate(int first, int middle, int last) { int l = middle - first; int r = last - middle; while ((l > 0) && (r > 0)) { if (l == r) { ssBlockSwap(first, middle, l); break; } if (l < r) { int a = last - 1; int b = middle - 1; int t = _sa[a]; while (true) { _sa[a--] = _sa[b]; _sa[b--] = _sa[a]; if (b < first) { _sa[a] = t; last = a; r -= (l + 1); if (r <= l) break; a--; b = middle - 1; t = _sa[a]; } } } else { int a = first; int b = middle; int t = _sa[a]; while (true) { _sa[a++] = _sa[b]; _sa[b++] = _sa[a]; if (last <= b) { _sa[a] = t; first = a + 1; l -= (r + 1); if (l <= r) break; a++; b = middle; t = _sa[a]; } } } } } void DivSufSort::ssSwapMerge(int pa, int first, int middle, int last, int buf, int bufSize, int depth) { int check = 0; while (true) { if (last - middle <= bufSize) { if ((first < middle) && (middle < last)) ssMergeBackward(pa, first, middle, last, buf, depth); if (((check & 1) != 0) || (((check & 2) != 0) && (ssCompare(&_sa[pa + getIndex(_sa[first - 1])], &_sa[pa + _sa[first]], depth) == 0))) { _sa[first] = ~_sa[first]; } if (((check & 4) != 0) && (ssCompare(&_sa[pa + getIndex(_sa[last - 1])], &_sa[pa + _sa[last]], depth) == 0)) { _sa[last] = ~_sa[last]; } const StackElement* se = _mergeStack->pop(); if (se == nullptr) return; first = se->_a; middle = se->_b; last = se->_c; check = se->_d; continue; } if (middle - first <= bufSize) { if (first < middle) ssMergeForward(pa, first, middle, last, buf, depth); if (((check & 1) != 0) || (((check & 2) != 0) && (ssCompare(&_sa[pa + getIndex(_sa[first - 1])], &_sa[pa + _sa[first]], depth) == 0))) { _sa[first] = ~_sa[first]; } if (((check & 4) != 0) && (ssCompare(&_sa[pa + getIndex(_sa[last - 1])], &_sa[pa + _sa[last]], depth) == 0)) { _sa[last] = ~_sa[last]; } const StackElement* se = _mergeStack->pop(); if (se == nullptr) return; first = se->_a; middle = se->_b; last = se->_c; check = se->_d; continue; } int len = (middle - first < last - middle) ? middle - first : last - middle; int m = 0; for (int half = len >> 1; len > 0; len = half, half >>= 1) { if (ssCompare(&_sa[pa + getIndex(_sa[middle + m + half])], &_sa[pa + getIndex(_sa[middle - m - half - 1])], depth) < 0) { m += (half + 1); half -= ((len & 1) ^ 1); } } if (m > 0) { int lm = middle - m; int rm = middle + m; ssBlockSwap(lm, middle, m); int l = middle; int r = l; int next = 0; if (rm < last) { if (_sa[rm] < 0) { _sa[rm] = ~_sa[rm]; if (first < lm) { l--; while (_sa[l] < 0) l--; next |= 4; } next |= 1; } else if (first < lm) { while (_sa[r] < 0) r++; next |= 2; } } if (l - first <= last - r) { _mergeStack->push(r, rm, last, (next & 3) | (check & 4), 0); middle = lm; last = l; check = (check & 3) | (next & 4); } else { if ((r == middle) && ((next & 2) != 0)) next ^= 6; _mergeStack->push(first, lm, l, (check & 3) | (next & 4), 0); first = r; middle = rm; check = (next & 3) | (check & 4); } } else { if (ssCompare(&_sa[pa + getIndex(_sa[middle - 1])], &_sa[pa + _sa[middle]], depth) == 0) { _sa[middle] = ~_sa[middle]; } if (((check & 1) != 0) || (((check & 2) != 0) && (ssCompare(&_sa[pa + getIndex(_sa[first - 1])], &_sa[pa + _sa[first]], depth) == 0))) { _sa[first] = ~_sa[first]; } if (((check & 4) != 0) && (ssCompare(&_sa[pa + getIndex(_sa[last - 1])], &_sa[pa + _sa[last]], depth) == 0)) { _sa[last] = ~_sa[last]; } const StackElement* se = _mergeStack->pop(); if (se == nullptr) return; first = se->_a; middle = se->_b; last = se->_c; check = se->_d; } } } void DivSufSort::ssMergeForward(int pa, int first, int middle, int last, int buf, int depth) { const int bufEnd = buf + middle - first - 1; ssBlockSwap(buf, first, middle - first); int a = first; int b = buf; int c = middle; const int t = _sa[a]; while (true) { const int r = ssCompare(&_sa[pa + _sa[b]], &_sa[pa + _sa[c]], depth); if (r < 0) { do { _sa[a++] = _sa[b]; if (bufEnd <= b) { _sa[bufEnd] = t; return; } _sa[b++] = _sa[a]; } while (_sa[b] < 0); } else if (r > 0) { do { _sa[a++] = _sa[c]; _sa[c++] = _sa[a]; if (last <= c) { while (b < bufEnd) { _sa[a++] = _sa[b]; _sa[b++] = _sa[a]; } _sa[a] = _sa[b]; _sa[b] = t; return; } } while (_sa[c] < 0); } else { _sa[c] = ~_sa[c]; do { _sa[a++] = _sa[b]; if (bufEnd <= b) { _sa[bufEnd] = t; return; } _sa[b++] = _sa[a]; } while (_sa[b] < 0); do { _sa[a++] = _sa[c]; _sa[c++] = _sa[a]; if (last <= c) { while (b < bufEnd) { _sa[a++] = _sa[b]; _sa[b++] = _sa[a]; } _sa[a] = _sa[b]; _sa[b] = t; return; } } while (_sa[c] < 0); } } } void DivSufSort::ssMergeBackward(int pa, int first, int middle, int last, int buf, int depth) { const int bufEnd = buf + last - middle - 1; ssBlockSwap(buf, middle, last - middle); int x = 0; int p1, p2; if (_sa[bufEnd] < 0) { p1 = pa + ~_sa[bufEnd]; x |= 1; } else p1 = pa + _sa[bufEnd]; if (_sa[middle - 1] < 0) { p2 = pa + ~_sa[middle - 1]; x |= 2; } else p2 = pa + _sa[middle - 1]; int a = last - 1; int b = bufEnd; int c = middle - 1; const int t = _sa[a]; while (true) { const int r = ssCompare(&_sa[p1], &_sa[p2], depth); if (r > 0) { if ((x & 1) != 0) { do { _sa[a--] = _sa[b]; _sa[b--] = _sa[a]; } while (_sa[b] < 0); x ^= 1; } _sa[a--] = _sa[b]; if (b <= buf) { _sa[buf] = t; break; } _sa[b--] = _sa[a]; if (_sa[b] < 0) { p1 = pa + ~_sa[b]; x |= 1; } else p1 = pa + _sa[b]; } else if (r < 0) { if ((x & 2) != 0) { do { _sa[a--] = _sa[c]; _sa[c--] = _sa[a]; } while (_sa[c] < 0); x ^= 2; } _sa[a--] = _sa[c]; _sa[c--] = _sa[a]; if (c < first) { while (buf < b) { _sa[a--] = _sa[b]; _sa[b--] = _sa[a]; } _sa[a] = _sa[b]; _sa[b] = t; break; } if (_sa[c] < 0) { p2 = pa + ~_sa[c]; x |= 2; } else p2 = pa + _sa[c]; } else // r = 0 { if ((x & 1) != 0) { do { _sa[a--] = _sa[b]; _sa[b--] = _sa[a]; } while (_sa[b] < 0); x ^= 1; } _sa[a--] = ~_sa[b]; if (b <= buf) { _sa[buf] = t; break; } _sa[b--] = _sa[a]; if ((x & 2) != 0) { do { _sa[a--] = _sa[c]; _sa[c--] = _sa[a]; } while (_sa[c] < 0); x ^= 2; } _sa[a--] = _sa[c]; _sa[c--] = _sa[a]; if (c < first) { while (buf < b) { _sa[a--] = _sa[b]; _sa[b--] = _sa[a]; } _sa[a] = _sa[b]; _sa[b] = t; break; } if (_sa[b] < 0) { p1 = pa + ~_sa[b]; x |= 1; } else p1 = pa + _sa[b]; if (_sa[c] < 0) { p2 = pa + ~_sa[c]; x |= 2; } else p2 = pa + _sa[c]; } } } void DivSufSort::ssInsertionSort(const int pa, int first, int last, int depth) { for (int i = last - 2; i >= first; i--) { const int t = pa + _sa[i]; int j = i + 1; int r; while ((r = ssCompare(&_sa[t], &_sa[pa + _sa[j]], depth)) > 0) { do { _sa[j - 1] = _sa[j]; j++; } while ((j < last) && (_sa[j] < 0)); if (j >= last) break; } _sa[j] = r == 0 ? ~_sa[j] : _sa[j]; _sa[j - 1] = t - pa; } } void DivSufSort::ssMultiKeyIntroSort(int pa, int first, int last, int depth) { const int* sapa = &_sa[pa]; int limit = ssIlg(last - first); int x = 0; while (true) { if (last - first <= SS_INSERTIONSORT_THRESHOLD) { if (last - first > 1) ssInsertionSort(pa, first, last, depth); const StackElement* se = _ssStack->pop(); if (se == nullptr) return; first = se->_a; last = se->_b; depth = se->_c; limit = se->_d; continue; } const int idx = depth; const uint8* p = &_buffer[idx]; if (limit == 0) ssHeapSort(idx, pa, first, last - first); limit--; int a; if (limit < 0) { int v = p[sapa[_sa[first]]]; for (a = first + 1; a < last; a++) { if ((x = p[sapa[_sa[a]]]) != v) { if (a - first > 1) break; v = x; first = a; } } if (p[sapa[_sa[first]] - 1] < v) first = ssPartition(pa, first, a, depth); if (a - first <= last - a) { if (a - first > 1) { _ssStack->push(a, last, depth, -1, 0); last = a; depth++; limit = ssIlg(a - first); } else { first = a; limit = -1; } } else { if (last - a > 1) { _ssStack->push(first, a, depth + 1, ssIlg(a - first), 0); first = a; limit = -1; } else { last = a; depth++; limit = ssIlg(a - first); } } continue; } // choose pivot a = ssPivot(idx, pa, first, last); const int v = p[sapa[_sa[a]]]; std::swap(_sa[first], _sa[a]); int b = first; // partition while (++b < last) { if ((x = p[sapa[_sa[b]]]) != v) break; } a = b; if ((a < last) && (x < v)) { while (++b < last) { if ((x = p[sapa[_sa[b]]]) > v) break; if (x == v) { std::swap(_sa[b], _sa[a]); a++; } } } int c = last; while (--c > b) { if ((x = p[sapa[_sa[c]]]) != v) break; } int d = c; if ((b < d) && (x > v)) { while (--c > b) { if ((x = p[sapa[_sa[c]]]) < v) break; if (x == v) { std::swap(_sa[c], _sa[d]); d--; } } } while (b < c) { std::swap(_sa[b], _sa[c]); while (++b < c) { if ((x = p[sapa[_sa[b]]]) > v) break; if (x == v) { std::swap(_sa[b], _sa[a]); a++; } } while (--c > b) { if ((x = p[sapa[_sa[c]]]) < v) break; if (x == v) { std::swap(_sa[c], _sa[d]); d--; } } } if (a <= d) { c = b - 1; int s = (a - first > b - a) ? b - a : a - first; for (int e = first, f = b - s; s > 0; s--, e++, f++) std::swap(_sa[e], _sa[f]); s = (d - c > last - d - 1) ? last - d - 1 : d - c; for (int e = b, f = last - s; s > 0; s--, e++, f++) std::swap(_sa[e], _sa[f]); a = first + (b - a); c = last - (d - c); b = (v <= p[sapa[_sa[a]] - 1]) ? a : ssPartition(pa, a, c, depth); if (a - first <= last - c) { if (last - c <= c - b) { _ssStack->push(b, c, depth + 1, ssIlg(c - b), 0); _ssStack->push(c, last, depth, limit, 0); last = a; } else if (a - first <= c - b) { _ssStack->push(c, last, depth, limit, 0); _ssStack->push(b, c, depth + 1, ssIlg(c - b), 0); last = a; } else { _ssStack->push(c, last, depth, limit, 0); _ssStack->push(first, a, depth, limit, 0); first = b; last = c; depth++; limit = ssIlg(c - b); } } else { if (a - first <= c - b) { _ssStack->push(b, c, depth + 1, ssIlg(c - b), 0); _ssStack->push(first, a, depth, limit, 0); first = c; } else if (last - c <= c - b) { _ssStack->push(first, a, depth, limit, 0); _ssStack->push(b, c, depth + 1, ssIlg(c - b), 0); first = c; } else { _ssStack->push(first, a, depth, limit, 0); _ssStack->push(c, last, depth, limit, 0); first = b; last = c; depth++; limit = ssIlg(c - b); } } } else { if (p[sapa[_sa[first]] - 1] < v) { first = ssPartition(pa, first, last, depth); limit = ssIlg(last - first); } else { limit++; } depth++; } } } int DivSufSort::ssPivot(int td, int pa, int first, int last) const { int t = last - first; int middle = first + (t >> 1); if (t <= 512) { return (t <= 32) ? ssMedian3(&_buffer[td], pa, first, middle, last - 1) : ssMedian5(&_buffer[td], pa, first, first + (t >> 2), middle, last - 1 - (t >> 2), last - 1); } t >>= 3; first = ssMedian3(&_buffer[td], pa, first, first + t, first + (t << 1)); middle = ssMedian3(&_buffer[td], pa, middle - t, middle, middle + t); last = ssMedian3(&_buffer[td], pa, last - 1 - (t << 1), last - 1 - t, last - 1); return ssMedian3(&_buffer[td], pa, first, middle, last); } int DivSufSort::ssMedian5(const uint8 buf0[], int pa, int v1, int v2, int v3, int v4, int v5) const { const int* buf1 = &_sa[pa]; if (buf0[buf1[_sa[v2]]] > buf0[buf1[_sa[v3]]]) { std::swap(v2, v3); } if (buf0[buf1[_sa[v4]]] > buf0[buf1[_sa[v5]]]) { std::swap(v4, v5); } if (buf0[buf1[_sa[v2]]] > buf0[buf1[_sa[v4]]]) { //std::swap(v2, v4); v4 = v2; std::swap(v3, v5); } if (buf0[buf1[_sa[v1]]] > buf0[buf1[_sa[v3]]]) { std::swap(v1, v3); } if (buf0[buf1[_sa[v1]]] > buf0[buf1[_sa[v4]]]) { //std::swap(v1, v4); v4 = v1; //std::swap(v3, v5); v3 = v5; } return (buf0[buf1[_sa[v3]]] > buf0[buf1[_sa[v4]]]) ? v4 : v3; } int DivSufSort::ssMedian3(const uint8 buf0[], int pa, int v1, int v2, int v3) const { const int* buf1 = &_sa[pa]; if (buf0[buf1[_sa[v1]]] > buf0[buf1[_sa[v2]]]) { std::swap(v1, v2); } if (buf0[buf1[_sa[v2]]] > buf0[buf1[_sa[v3]]]) { return (buf0[buf1[_sa[v1]]] > buf0[buf1[_sa[v3]]]) ? v1 : v3; } return v2; } int DivSufSort::ssPartition(int pa, int first, int last, int depth) { int a = first - 1; int b = last; const int d = depth - 1; const int pb = pa + 1; while (true) { a++; while ((a < b) && (_sa[pa + _sa[a]] + d >= _sa[pb + _sa[a]])) { _sa[a] = ~_sa[a]; a++; } b--; while ((b > a) && (_sa[pa + _sa[b]] + d < _sa[pb + _sa[b]])) b--; if (b <= a) break; const int t = ~_sa[b]; _sa[b] = _sa[a]; _sa[a] = t; } if (first < a) _sa[first] = ~_sa[first]; return a; } void DivSufSort::ssHeapSort(int idx, int pa, int saIdx, int size) { int m = size; if ((size & 1) == 0) { m--; if (_buffer[idx + _sa[pa + _sa[saIdx + (m >> 1)]]] < _buffer[idx + _sa[pa + _sa[saIdx + m]]]) std::swap(_sa[saIdx + m], _sa[saIdx + (m >> 1)]); } for (int i = (m >> 1) - 1; i >= 0; i--) ssFixDown(idx, pa, saIdx, i, m); if ((size & 1) == 0) { std::swap(_sa[saIdx], _sa[saIdx + m]); ssFixDown(idx, pa, saIdx, 0, m); } for (int i = m - 1; i > 0; i--) { const int t = _sa[saIdx]; _sa[saIdx] = _sa[saIdx + i]; ssFixDown(idx, pa, saIdx, 0, i); _sa[saIdx + i] = t; } } void DivSufSort::ssFixDown(int idx, int pa, int saIdx, int i, int size) { const int v = _sa[saIdx + i]; const int c = _buffer[idx + _sa[pa + v]]; int j = (i << 1) + 1; while (j < size) { int k = j; j++; int d = _buffer[idx + _sa[pa + _sa[saIdx + k]]]; const int e = _buffer[idx + _sa[pa + _sa[saIdx + j]]]; if (d < e) { k = j; d = e; } if (d <= c) break; _sa[saIdx + i] = _sa[saIdx + k]; i = k; j = (i << 1) + 1; } _sa[i + saIdx] = v; } // Tandem Repeat Sort void DivSufSort::trSort(int n, int depth) { TRBudget budget(trIlg(n) * 2 / 3, n); for (int isad = n + depth; _sa[0] > -n; isad += (isad - n)) { int first = 0; int skip = 0; int unsorted = 0; do { const int t = _sa[first]; if (t < 0) { first -= t; skip += t; continue; } if (skip != 0) { _sa[first + skip] = skip; skip = 0; } const int last = _sa[n + t] + 1; if (last - first > 1) { budget._count = 0; trIntroSort(n, isad, first, last, budget); if (budget._count != 0) unsorted += budget._count; else skip = first - last; } else if (last - first == 1) skip = -1; first = last; } while (first < n); if (skip != 0) _sa[first + skip] = skip; if (unsorted == 0) break; } } uint64 DivSufSort::trPartition(int isad, int first, int middle, int last, int v) { int x = 0; int b = middle; const int* p = &_sa[isad]; while (b < last) { x = p[ _sa[b]]; if (x != v) break; b++; } int a = b; if ((a < last) && (x < v)) { while (++b < last) { if ((x = p[_sa[b]]) > v) break; if (x == v) { std::swap(_sa[a], _sa[b]); a++; } } } int c = last - 1; while (c > b) { x = p[_sa[c]]; if (x != v) break; c--; } int d = c; if ((b < d) && (x > v)) { while (--c > b) { if ((x = p[_sa[c]]) < v) break; if (x == v) { std::swap(_sa[c], _sa[d]); d--; } } } while (b < c) { std::swap(_sa[c], _sa[b]); while ((++b < c) && ((x = p[_sa[b]]) <= v)) { if (x == v) { std::swap(_sa[a], _sa[b]); a++; } } while ((--c > b) && ((x = p[_sa[c]]) >= v)) { if (x == v) { std::swap(_sa[c], _sa[d]); d--; } } } if (a <= d) { c = b - 1; int s = a - first; if (s > b - a) s = b - a; for (int e = first, f = b - s; s > 0; s--, e++, f++) std::swap(_sa[e], _sa[f]); s = d - c; if (s >= last - d) s = last - d - 1; for (int e = b, f = last - s; s > 0; s--, e++, f++) std::swap(_sa[e], _sa[f]); first += (b - a); last -= (d - c); } return ((uint64(first) << 32) | (uint64(last) & uint64(0xFFFFFFFF))); } void DivSufSort::trIntroSort(int isa, int isad, int first, int last, TRBudget& budget) { const int incr = isad - isa; int limit = trIlg(last - first); int trlink = -1; while (true) { if (limit < 0) { if (limit == -1) { // tandem repeat partition uint64 res = trPartition(isad - incr, first, first, last, last - 1); const int a = int(res >> 32); const int b = int(res); // update ranks if (a < last) { for (int c = first, v = a - 1; c < a; c++) _sa[isa + _sa[c]] = v; } if (b < last) { for (int c = a, v = b - 1; c < b; c++) _sa[isa + _sa[c]] = v; } // push if (b - a > 1) { _trStack->push(0, a, b, 0, 0); _trStack->push(isad - incr, first, last, -2, trlink); trlink = _trStack->size() - 2; } if (a - first <= last - b) { if (a - first > 1) { _trStack->push(isad, b, last, trIlg(last - b), trlink); last = a; limit = trIlg(a - first); } else if (last - b > 1) { first = b; limit = trIlg(last - b); } else { const StackElement* se = _trStack->pop(); if (se == nullptr) return; isad = se->_a; first = se->_b; last = se->_c; limit = se->_d; trlink = se->_e; } } else { if (last - b > 1) { _trStack->push(isad, first, a, trIlg(a - first), trlink); first = b; limit = trIlg(last - b); } else if (a - first > 1) { last = a; limit = trIlg(a - first); } else { const StackElement* se = _trStack->pop(); if (se == nullptr) return; isad = se->_a; first = se->_b; last = se->_c; limit = se->_d; trlink = se->_e; } } } else if (limit == -2) { // tandem repeat copy const StackElement* se = _trStack->pop(); if (se == nullptr) return; if (se->_d == 0) { trCopy(isa, first, se->_b, se->_c, last, isad - isa); } else { if (trlink >= 0) _trStack->get(trlink)->_d = -1; trPartialCopy(isa, first, se->_b, se->_c, last, isad - isa); } se = _trStack->pop(); if (se == nullptr) return; isad = se->_a; first = se->_b; last = se->_c; limit = se->_d; trlink = se->_e; } else { // sorted partition if (_sa[first] >= 0) { int a = first; do { _sa[isa + _sa[a]] = a; a++; } while ((a < last) && (_sa[a] >= 0)); first = a; } if (first < last) { int a = first; do { _sa[a] = ~_sa[a]; a++; } while (_sa[a] < 0); int next = (_sa[isa + _sa[a]] != _sa[isad + _sa[a]]) ? trIlg(a - first + 1) : -1; a++; if (a < last) { const int v = a - 1; for (int b = first; b < a; b++) _sa[isa + _sa[b]] = v; } // push if (budget.check(a - first) == true) { if (a - first <= last - a) { _trStack->push(isad, a, last, -3, trlink); isad += incr; last = a; limit = next; } else { if (last - a > 1) { _trStack->push(isad + incr, first, a, next, trlink); first = a; limit = -3; } else { isad += incr; last = a; limit = next; } } } else { if (trlink >= 0) _trStack->get(trlink)->_d = -1; if (last - a > 1) { first = a; limit = -3; } else { const StackElement* se = _trStack->pop(); if (se == nullptr) return; isad = se->_a; first = se->_b; last = se->_c; limit = se->_d; trlink = se->_e; } } } else { const StackElement* se = _trStack->pop(); if (se == nullptr) return; isad = se->_a; first = se->_b; last = se->_c; limit = se->_d; trlink = se->_e; } } continue; } if (last - first <= TR_INSERTIONSORT_THRESHOLD) { trInsertionSort(&_sa[isad], first, last); limit = -3; continue; } if (limit == 0) { trHeapSort(isad, first, last - first); int a = last - 1; while (first < a) { int b = a - 1; for (int x = _sa[isad + _sa[a]]; (first <= b) && (_sa[isad + _sa[b]] == x); b--) _sa[b] = ~_sa[b]; a = b; } limit = -3; continue; } limit--; // choose pivot std::swap(_sa[first], _sa[trPivot(_sa, isad, first, last)]); int v = _sa[isad + _sa[first]]; // partition uint64 res = trPartition(isad, first, first + 1, last, v); const int a = int(res >> 32); const int b = int(res & 0xFFFFFFFFL); if (last - first != b - a) { const int next = (_sa[isa + _sa[a]] != v) ? trIlg(b - a) : -1; v = a - 1; // update ranks for (int c = first; c < a; c++) _sa[isa + _sa[c]] = v; if (b < last) { v = b - 1; for (int c = a; c < b; c++) _sa[isa + _sa[c]] = v; } // push if ((b - a > 1) && (budget.check(b - a) == true)) { if (a - first <= last - b) { if (last - b <= b - a) { if (a - first > 1) { _trStack->push(isad + incr, a, b, next, trlink); _trStack->push(isad, b, last, limit, trlink); last = a; } else if (last - b > 1) { _trStack->push(isad + incr, a, b, next, trlink); first = b; } else { isad += incr; first = a; last = b; limit = next; } } else if (a - first <= b - a) { if (a - first > 1) { _trStack->push(isad, b, last, limit, trlink); _trStack->push(isad + incr, a, b, next, trlink); last = a; } else { _trStack->push(isad, b, last, limit, trlink); isad += incr; first = a; last = b; limit = next; } } else { _trStack->push(isad, b, last, limit, trlink); _trStack->push(isad, first, a, limit, trlink); isad += incr; first = a; last = b; limit = next; } } else { if (a - first <= b - a) { if (last - b > 1) { _trStack->push(isad + incr, a, b, next, trlink); _trStack->push(isad, first, a, limit, trlink); first = b; } else if (a - first > 1) { _trStack->push(isad + incr, a, b, next, trlink); last = a; } else { isad += incr; first = a; last = b; limit = next; } } else if (last - b <= b - a) { if (last - b > 1) { _trStack->push(isad, first, a, limit, trlink); _trStack->push(isad + incr, a, b, next, trlink); first = b; } else { _trStack->push(isad, first, a, limit, trlink); isad += incr; first = a; last = b; limit = next; } } else { _trStack->push(isad, first, a, limit, trlink); _trStack->push(isad, b, last, limit, trlink); isad += incr; first = a; last = b; limit = next; } } } else { if ((b - a > 1) && (trlink >= 0)) _trStack->get(trlink)->_d = -1; if (a - first <= last - b) { if (a - first > 1) { _trStack->push(isad, b, last, limit, trlink); last = a; } else if (last - b > 1) { first = b; } else { const StackElement* se = _trStack->pop(); if (se == nullptr) return; isad = se->_a; first = se->_b; last = se->_c; limit = se->_d; trlink = se->_e; } } else { if (last - b > 1) { _trStack->push(isad, first, a, limit, trlink); first = b; } else if (a - first > 1) { last = a; } else { const StackElement* se = _trStack->pop(); if (se == nullptr) return; isad = se->_a; first = se->_b; last = se->_c; limit = se->_d; trlink = se->_e; } } } } else { if (budget.check(last - first) == true) { limit = trIlg(last - first); isad += incr; } else { if (trlink >= 0) _trStack->get(trlink)->_d = -1; const StackElement* se = _trStack->pop(); if (se == nullptr) return; isad = se->_a; first = se->_b; last = se->_c; limit = se->_d; trlink = se->_e; } } } } int DivSufSort::trPivot(const int arr[], int isad, int first, int last) const { int t = last - first; int middle = first + (t >> 1); if (t <= 512) { if (t <= 32) return trMedian3(arr, isad, first, middle, last - 1); t >>= 2; return trMedian5(arr, isad, first, first + t, middle, last - 1 - t, last - 1); } t >>= 3; first = trMedian3(arr, isad, first, first + t, first + (t << 1)); middle = trMedian3(arr, isad, middle - t, middle, middle + t); last = trMedian3(arr, isad, last - 1 - (t << 1), last - 1 - t, last - 1); return trMedian3(arr, isad, first, middle, last); } void DivSufSort::trHeapSort(int isad, int saIdx, int size) { int m = size; if ((size & 1) == 0) { m--; if (_sa[isad + _sa[saIdx + (m >> 1)]] < _sa[isad + _sa[saIdx + m]]) std::swap(_sa[saIdx + m], _sa[saIdx + (m >> 1)]); } for (int i = (m >> 1) - 1; i >= 0; i--) trFixDown(isad, saIdx, i, m); if ((size & 1) == 0) { std::swap(_sa[saIdx], _sa[saIdx + m]); trFixDown(isad, saIdx, 0, m); } for (int i = m - 1; i > 0; i--) { const int t = _sa[saIdx]; _sa[saIdx] = _sa[saIdx + i]; trFixDown(isad, saIdx, 0, i); _sa[saIdx + i] = t; } } void DivSufSort::trFixDown(int isad, int saIdx, int i, int size) { const int v = _sa[saIdx + i]; const int c = _sa[isad + v]; int j = (i << 1) + 1; while (j < size) { int k = j; j++; int d = _sa[isad + _sa[saIdx + k]]; const int e = _sa[isad + _sa[saIdx + j]]; if (d < e) { k = j; d = e; } if (d <= c) break; _sa[saIdx + i] = _sa[saIdx + k]; i = k; j = (i << 1) + 1; } _sa[saIdx + i] = v; } void DivSufSort::trInsertionSort(const int arr[], int first, int last) { for (int a = first + 1; a < last; a++) { int b = a - 1; const int t = _sa[a]; int r; while ((r = arr[t] - arr[_sa[b]]) < 0) { do { _sa[b + 1] = _sa[b]; b--; } while ((b >= first) && (_sa[b] < 0)); if (b < first) break; } if (r == 0) _sa[b] = ~_sa[b]; _sa[b + 1] = t; } } void DivSufSort::trPartialCopy(int isa, int first, int a, int b, int last, int depth) { const int v = b - 1; int lastRank = -1; int newRank = -1; int d = a - 1; for (int c = first; c <= d; c++) { const int s = _sa[c] - depth; if ((s >= 0) && (_sa[isa + s] == v)) { d++; _sa[d] = s; const int rank = _sa[isa + s + depth]; if (lastRank != rank) { lastRank = rank; newRank = d; } _sa[isa + s] = newRank; } } lastRank = -1; for (int e = d; first <= e; e--) { const int rank = _sa[isa + _sa[e]]; if (lastRank != rank) { lastRank = rank; newRank = e; } if (newRank != rank) { _sa[isa + _sa[e]] = newRank; } } lastRank = -1; const int e = d + 1; d = b; for (int c = last - 1; d > e; c--) { const int s = _sa[c] - depth; if ((s >= 0) && (_sa[isa + s] == v)) { d--; _sa[d] = s; const int rank = _sa[isa + s + depth]; if (lastRank != rank) { lastRank = rank; newRank = d; } _sa[isa + s] = newRank; } } } void DivSufSort::trCopy(int isa, int first, int a, int b, int last, int depth) { const int v = b - 1; int d = a - 1; for (int c = first; c <= d; c++) { const int s = _sa[c] - depth; if ((s >= 0) && (_sa[isa + s] == v)) { d++; _sa[d] = s; _sa[isa + s] = d; } } const int e = d + 1; d = b; for (int c = last - 1; d > e; c--) { const int s = _sa[c] - depth; if ((s >= 0) && (_sa[isa + s] == v)) { d--; _sa[d] = s; _sa[isa + s] = d; } } } kanzi-cpp-2.6.0/src/transform/DivSufSort.hpp000066400000000000000000000213331525576310500210110ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_DivSufSort #define knz_DivSufSort #include "../types.hpp" #if __cplusplus >= 201103L #include #else #include #endif namespace kanzi { // DivSufSort is a fast two-stage suffix sorting algorithm by Yuta Mori. // The original C code is here: https://code.google.com/p/libdivsufsort/ // See also https://code.google.com/p/libdivsufsort/source/browse/wiki/SACA_Benchmarks.wiki // for comparison of different suffix array construction algorithms. // It is used to implement the forward stage of the BWT in linear time. struct StackElement { int _a, _b, _c, _d, _e; StackElement() { _a = _b = _c = _d = _e = 0; } }; // A stack of pre-allocated elements class Stack { friend class DivSufSort; private: StackElement* _arr; int _index; Stack(int size) { _arr = new StackElement[size]; _index = 0; } ~Stack() { delete[] _arr; } StackElement* get(int idx) const { return &_arr[idx]; } int size() const { return _index; } void push(int a, int b, int c, int d, int e) { StackElement* elt = &_arr[_index]; elt->_a = a; elt->_b = b; elt->_c = c; elt->_d = d; elt->_e = e; _index++; } StackElement* pop() { return (_index == 0) ? nullptr : &_arr[--_index]; } }; struct TRBudget { int _chance; int _remain; int _incVal; int _count; TRBudget(int chance, int incval) : _chance(chance) , _remain(incval) { _incVal = incval; _count = 0; } bool check(int size); }; inline bool TRBudget::check(int size) { if (size <= _remain) { _remain -= size; return true; } if (_chance == 0) { _count += size; return false; } _remain += (_incVal - size); _chance--; return true; } class DivSufSort { private: static const int SS_INSERTIONSORT_THRESHOLD; static const int SS_BLOCKSIZE; static const int SS_MISORT_STACKSIZE; static const int SS_SMERGE_STACKSIZE; static const int TR_STACKSIZE; static const int TR_INSERTIONSORT_THRESHOLD; static const int SQQ_TABLE[]; static const int LOG_TABLE[]; int* _sa; const uint8* _buffer; Stack* _ssStack; Stack* _trStack; Stack* _mergeStack; int _bucketA[256]; int _bucketB[65536]; void constructSuffixArray(int bucketA[], int bucketB[], int n, int m); int constructBWT(int bucketA[], int bucketB[], int n, int m, int indexes[], int idxCount); int sortTypeBstar(int bucketA[], int bucketB[], int n); void ssSort(int pa, int first, int last, int buf, int bufSize, int depth, int n, bool lastSuffix); int ssCompare(int pa, int pb, int p2, const int depth) const; int ssCompare(const int s1[], const int s2[], const int depth) const; void ssInplaceMerge(int pa, int first, int middle, int last, int depth); void ssRotate(int first, int middle, int last); void ssBlockSwap(int a, int b, int n); static int getIndex(int a) { return (a >= 0) ? a : ~a; } void ssSwapMerge(int pa, int first, int middle, int last, int buf, int bufSize, int depth); void ssMergeForward(int pa, int first, int middle, int last, int buf, int depth); void ssMergeBackward(int pa, int first, int middle, int last, int buf, int depth); void ssInsertionSort(int pa, int first, int last, int depth); int ssIsqrt(int x) const; void ssMultiKeyIntroSort(int pa, int first, int last, int depth); int ssPivot(int td, int pa, int first, int last) const; int ssMedian5(const uint8 buf[], int pa, int v1, int v2, int v3, int v4, int v5) const; int ssMedian3(const uint8 buf[], int pa, int v1, int v2, int v3) const; int ssPartition(int pa, int first, int last, int depth); void ssHeapSort(int idx, int pa, int saIdx, int size); void ssFixDown(int idx, int pa, int saIdx, int i, int size); static int ssIlg(int n); void trSort(int n, int depth); uint64 trPartition(int isad, int first, int middle, int last, int v); void trIntroSort(int isa, int isad, int first, int last, TRBudget& budget); int trPivot(const int arr[], int isad, int first, int last) const; int trMedian5(const int arr[], int isad, int v1, int v2, int v3, int v4, int v5) const; int trMedian3(const int arr[], int isad, int v1, int v2, int v3) const; void trHeapSort(int isad, int saIdx, int size); void trFixDown(int isad, int saIdx, int i, int size); void trInsertionSort(const int arr[], int first, int last); void trPartialCopy(int isa, int first, int a, int b, int last, int depth); void trCopy(int isa, int first, int a, int b, int last, int depth); void reset(); int trIlg(int n) const; public: DivSufSort(); ~DivSufSort(); bool computeSuffixArray(const byte input[], int sa[], int length); bool computeBWT(const byte input[], byte output[], int sa[], int length, int indexes[], int idxCount = 8); }; inline int DivSufSort::ssIlg(int n) { return (n > 255) ? 8 + LOG_TABLE[n >> 8] : LOG_TABLE[n & 0xFF]; } inline void DivSufSort::ssBlockSwap(int a, int b, int n) { while (n-- > 0) { std::swap(_sa[a], _sa[b]); a++; b++; } } inline int DivSufSort::trIlg(int n) const { return ((n & 0xFFFF0000) != 0) ? (((n & 0xFF000000) != 0) ? 24 + LOG_TABLE[(n >> 24) & 0xFF] : 16 + LOG_TABLE[(n >> 16) & 0xFF]) : (((n & 0x0000FF00) != 0) ? 8 + LOG_TABLE[(n >> 8) & 0xFF] : LOG_TABLE[n & 0xFF]); } inline int DivSufSort::trMedian5(const int sa[], int isad, int v1, int v2, int v3, int v4, int v5) const { if (sa[isad + sa[v2]] > sa[isad + sa[v3]]) { std::swap(v2, v3); } if (sa[isad + sa[v4]] > sa[isad + sa[v5]]) { const int t = v4; v4 = v5; v5 = t; } if (sa[isad + sa[v2]] > sa[isad + sa[v4]]) { std::swap(v2, v4); std::swap(v3, v5); } if (sa[isad + sa[v1]] > sa[isad + sa[v3]]) { std::swap(v1, v3); } if (sa[isad + sa[v1]] > sa[isad + sa[v4]]) { std::swap(v1, v4); std::swap(v3, v5); } if (sa[isad + sa[v3]] > sa[isad + sa[v4]]) return v4; return v3; } inline int DivSufSort::trMedian3(const int sa[], int isad, int v1, int v2, int v3) const { if (sa[isad + sa[v1]] > sa[isad + sa[v2]]) { std::swap(v1, v2); } if (sa[isad + sa[v2]] > sa[isad + sa[v3]]) { if (sa[isad + sa[v1]] > sa[isad + sa[v3]]) return v1; return v3; } return v2; } inline int DivSufSort::ssIsqrt(int x) const { if (x >= (SS_BLOCKSIZE * SS_BLOCKSIZE)) return SS_BLOCKSIZE; const int e = ((x & 0xFFFF0000) != 0) ? (((x & 0xFF000000) != 0) ? 24 + LOG_TABLE[(x >> 24) & 0xFF] : 16 + LOG_TABLE[(x >> 16) & 0xFF]) : (((x & 0x0000FF00) != 0) ? 8 + LOG_TABLE[(x >> 8) & 0xFF] : LOG_TABLE[x & 0xFF]); if (e < 8) return SQQ_TABLE[x] >> 4; int y; if (e >= 16) { y = SQQ_TABLE[x >> ((e - 6) - (e & 1))] << ((e >> 1) - 7); if (e >= 24) { y = (y + 1 + x / y) >> 1; } y = (y + 1 + x / y) >> 1; } else { y = (SQQ_TABLE[x >> ((e - 6) - (e & 1))] >> (7 - (e >> 1))) + 1; } return (x < y * y) ? y - 1 : y; } } #endif kanzi-cpp-2.6.0/src/transform/EXECodec.cpp000066400000000000000000000732611525576310500203220ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include "../Global.hpp" #include "../Magic.hpp" #include "EXECodec.hpp" using namespace kanzi; using namespace std; const kanzi::byte EXECodec::X86_MASK_JUMP = kanzi::byte(0xFE); const kanzi::byte EXECodec::X86_INSTRUCTION_JUMP = kanzi::byte(0xE8); const kanzi::byte EXECodec::X86_INSTRUCTION_JCC = kanzi::byte(0x80); const kanzi::byte EXECodec::X86_TWO_BYTE_PREFIX = kanzi::byte(0x0F); const kanzi::byte EXECodec::X86_MASK_JCC = kanzi::byte(0xF0); const kanzi::byte EXECodec::X86_ESCAPE = kanzi::byte(0x9B); const kanzi::byte EXECodec::NOT_EXE = kanzi::byte(0x80); const kanzi::byte EXECodec::X86 = kanzi::byte(0x40); const kanzi::byte EXECodec::ARM64 = kanzi::byte(0x20); const kanzi::byte EXECodec::MASK_DT = kanzi::byte(0x0F); const int EXECodec::X86_ADDR_MASK = (1 << 24) - 1; const int EXECodec::MASK_ADDRESS = 0xF0F0F0F0; const int EXECodec::ARM_B_ADDR_MASK = (1 << 26) - 1; const int EXECodec::ARM_B_OPCODE_MASK = 0xFFFFFFFF ^ ARM_B_ADDR_MASK; const int EXECodec::ARM_B_ADDR_SGN_MASK = 1 << 25; const int EXECodec::ARM_OPCODE_B = 0x14000000; // 6 bit opcode const int EXECodec::ARM_OPCODE_BL = 0x94000000; // 6 bit opcode const int EXECodec::ARM_CB_REG_BITS = 5; // lowest bits for register const int EXECodec::ARM_CB_ADDR_MASK = 0x00FFFFE0; // 18 bit addr mask const int EXECodec::ARM_CB_ADDR_SGN_MASK = 1 << 18; const int EXECodec::ARM_CB_OPCODE_MASK = 0x7F000000; const int EXECodec::ARM_OPCODE_CBZ = 0x34000000; // 8 bit opcode const int EXECodec::ARM_OPCODE_CBNZ = 0x35000000; // 8 bit opcode const int EXECodec::WIN_PE = 0x00004550; const uint16 EXECodec::WIN_X86_ARCH = 0x014C; const uint16 EXECodec::WIN_AMD64_ARCH = 0x8664; const uint16 EXECodec::WIN_ARM64_ARCH = 0xAA64; const int EXECodec::ELF_X86_ARCH = 0x03; const int EXECodec::ELF_AMD64_ARCH = 0x3E; const int EXECodec::ELF_ARM64_ARCH = 0xB7; const int EXECodec::MAC_AMD64_ARCH = 0x01000007; const int EXECodec::MAC_ARM64_ARCH = 0x0100000C; const int EXECodec::MAC_MH_EXECUTE = 0x02; const int EXECodec::MAC_LC_SEGMENT = 0x01; const int EXECodec::MAC_LC_SEGMENT64 = 0x19; const int EXECodec::MIN_BLOCK_SIZE = 4096; const int EXECodec::MAX_BLOCK_SIZE = (1 << (26 + 2)) - 1; // max offset << 2 bool EXECodec::forward(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if ((count < MIN_BLOCK_SIZE) || (count > MAX_BLOCK_SIZE)) return false; if (!SliceArray::isValid(input)) throw std::invalid_argument("EXECodec: Invalid input block"); if (!SliceArray::isValid(output)) throw std::invalid_argument("EXECodec: Invalid output block"); if (output._length - output._index < getMaxEncodedLength(count)) return false; if (_pCtx != nullptr) { Global::DataType dt = (Global::DataType)_pCtx->getInt("dataType", Global::UNDEFINED); if ((dt != Global::UNDEFINED) && (dt != Global::EXE) && (dt != Global::BIN)) return false; } int codeStart = 0; int codeEnd = count; kanzi::byte mode = detectType(&input._array[input._index], count, codeStart, codeEnd); if ((mode & NOT_EXE) != kanzi::byte(0)) { if (_pCtx != nullptr) _pCtx->putInt("dataType", Global::DataType(mode & MASK_DT)); return false; } mode &= ~MASK_DT; bool res = false; if (mode == X86) res = forwardX86(input, output, count, codeStart, codeEnd); else if (mode == ARM64) res = forwardARM(input, output, count, codeStart, codeEnd); else return false; if ((_pCtx != nullptr) && (res == true)) _pCtx->putInt("dataType", Global::EXE); return res; } bool EXECodec::forwardX86(SliceArray& input, SliceArray& output, int count, int codeStart, int codeEnd) { const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; const int dstCapacity = output._length - output._index; dst[0] = X86; int srcIdx = codeStart; int dstIdx = 9; int matches = 0; const int dstEnd = dstCapacity - 5; bool boundaryReached = false; if ((codeStart < 0) || (codeStart > count) || (dstIdx + codeStart > dstCapacity)) return false; if ((codeEnd < codeStart) || (codeEnd > count)) return false; if (codeStart > 0) { memcpy(&dst[dstIdx], &src[0], size_t(codeStart)); dstIdx += codeStart; } while ((srcIdx < codeEnd) && (dstIdx < dstEnd)) { if (src[srcIdx] == X86_TWO_BYTE_PREFIX) { if (srcIdx + 1 >= codeEnd) { boundaryReached = true; break; } if ((src[srcIdx + 1] & X86_MASK_JCC) == X86_INSTRUCTION_JCC) { if (srcIdx + 5 >= codeEnd) { boundaryReached = true; break; } } dst[dstIdx++] = src[srcIdx++]; if ((src[srcIdx] & X86_MASK_JCC) != X86_INSTRUCTION_JCC) { // Not a relative jump if (src[srcIdx] == X86_ESCAPE) dst[dstIdx++] = X86_ESCAPE; dst[dstIdx++] = src[srcIdx++]; continue; } if (srcIdx + 4 >= codeEnd) { boundaryReached = true; break; } } else if ((src[srcIdx] & X86_MASK_JUMP) != X86_INSTRUCTION_JUMP) { // Not a relative call if (src[srcIdx] == X86_ESCAPE) dst[dstIdx++] = X86_ESCAPE; dst[dstIdx++] = src[srcIdx++]; continue; } else if (srcIdx + 4 >= codeEnd) { boundaryReached = true; break; } // Current instruction is a jump/call. const int sgn = int(src[srcIdx + 4]); const int offset = LittleEndian::readInt32(&src[srcIdx + 1]); if (((sgn != 0) && (sgn != 0xFF)) || (offset == int(0xFF000000))) { dst[dstIdx++] = X86_ESCAPE; dst[dstIdx++] = src[srcIdx++]; continue; } // Absolute target address = srcIdx + 5 + offset. Let us ignore the +5 const int addr = srcIdx + ((sgn == 0) ? offset : -(-offset & X86_ADDR_MASK)); dst[dstIdx++] = src[srcIdx++]; BigEndian::writeInt32(&dst[dstIdx], addr ^ MASK_ADDRESS); srcIdx += 4; dstIdx += 4; matches++; } if ((matches < 16) || ((srcIdx < codeEnd) && (boundaryReached == false))) return false; if (dstIdx + (count - srcIdx) > dstEnd) return false; LittleEndian::writeInt32(&dst[1], codeStart); LittleEndian::writeInt32(&dst[5], dstIdx); memcpy(&dst[dstIdx], &src[srcIdx], size_t(count - srcIdx)); dstIdx += (count - srcIdx); // Cap expansion due to false positives if (dstIdx > count + (count / 50)) return false; input._index += count; output._index += dstIdx; return true; } bool EXECodec::forwardARM(SliceArray& input, SliceArray& output, int count, int codeStart, int codeEnd) { const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; const int dstCapacity = output._length - output._index; dst[0] = ARM64; int srcIdx = codeStart; int dstIdx = 9; int matches = 0; const int dstEnd = dstCapacity - 8; if ((codeStart < 0) || (codeStart > count) || (dstIdx + codeStart > dstCapacity)) return false; if ((codeEnd < codeStart) || (codeEnd > count)) return false; if (codeStart > 0) { memcpy(&dst[dstIdx], &src[0], size_t(codeStart)); dstIdx += codeStart; } while ((srcIdx + 4 <= codeEnd) && (dstIdx < dstEnd)) { const int instr = LittleEndian::readInt32(&src[srcIdx]); const int opcode1 = instr & ARM_B_OPCODE_MASK; //const int opcode2 = instr & ARM_CB_OPCODE_MASK; bool isBL = (opcode1 == ARM_OPCODE_B) || (opcode1 == ARM_OPCODE_BL); // unconditional jump bool isCB = false; // disable for now ... isCB = (opcode2 == ARM_OPCODE_CBZ) || (opcode2 == ARM_OPCODE_CBNZ); // conditional jump if ((isBL == false) && (isCB == false)) { // Not a relative jump memcpy(&dst[dstIdx], &src[srcIdx], 4); srcIdx += 4; dstIdx += 4; continue; } int addr, val; if (isBL == true) { // opcode(6) + sgn(1) + offset(25) // Absolute target address = srcIdx +/- (offset*4) const int offset = instr & ARM_B_ADDR_MASK; const int sgn = instr & ARM_B_ADDR_SGN_MASK; addr = srcIdx + 4 * ((sgn == 0) ? offset : -(-offset & ARM_B_ADDR_MASK)); if (addr < 0) addr = 0; val = opcode1 | (addr >> 2); } else { // isCB == true // opcode(8) + sgn(1) + offset(18) + register(5) // Absolute target address = srcIdx +/- (offset*4) const int offset = (instr & ARM_CB_ADDR_MASK) >> ARM_CB_REG_BITS; const int sgn = instr & ARM_CB_ADDR_SGN_MASK; addr = srcIdx + 4 * ((sgn == 0) ? offset : -(-offset & ARM_B_ADDR_MASK)); if (addr < 0) addr = 0; val = (instr & ~ARM_CB_ADDR_MASK) | ((addr >> 2) << ARM_CB_REG_BITS); } if (addr == 0) { LittleEndian::writeInt32(&dst[dstIdx], val); // 0 address as escape memcpy(&dst[dstIdx + 4], &src[srcIdx], 4); srcIdx += 4; dstIdx += 8; continue; } LittleEndian::writeInt32(&dst[dstIdx], val); srcIdx += 4; dstIdx += 4; matches++; } if ((matches < 16) || ((srcIdx + 4 <= codeEnd) && (dstIdx >= dstEnd))) return false; if (dstIdx + (count - srcIdx) > dstEnd) return false; LittleEndian::writeInt32(&dst[1], codeStart); LittleEndian::writeInt32(&dst[5], dstIdx); memcpy(&dst[dstIdx], &src[srcIdx], size_t(count - srcIdx)); dstIdx += (count - srcIdx); // Cap expansion due to false positives if (dstIdx > count + (count / 50)) return false; input._index += count; output._index += dstIdx; return true; } bool EXECodec::inverse(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if ((count < 9) || (count > input._length - input._index)) return false; if (!SliceArray::isValid(input)) throw std::invalid_argument("EXECodec: Invalid input block"); if (!SliceArray::isValid(output)) throw std::invalid_argument("EXECodec: Invalid output block"); kanzi::byte mode = input._array[input._index]; if (mode == X86) return inverseX86(input, output, count); if (mode == ARM64) return inverseARM(input, output, count); return false; } bool EXECodec::inverseX86(SliceArray& input, SliceArray& output, int count) { const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; int srcIdx = 9; int dstIdx = 0; const int dstEnd = output._length - output._index; const int codeStart = LittleEndian::readInt32(&src[1]); const int codeEnd = LittleEndian::readInt32(&src[5]); // Sanity check if ((codeStart < 0) || (codeEnd < srcIdx) || (codeEnd > count) || (codeStart > codeEnd - srcIdx) || (codeStart > dstEnd - dstIdx)) return false; if (codeStart > 0) { memcpy(&dst[dstIdx], &src[srcIdx], size_t(codeStart)); dstIdx += codeStart; srcIdx += codeStart; } while (srcIdx < codeEnd) { if (src[srcIdx] == X86_TWO_BYTE_PREFIX) { if (srcIdx + 1 >= codeEnd) { // Accept legacy streams where a trailing 0x0F was emitted in // the code section and the remaining bytes were copied as tail. if (dstIdx >= dstEnd) return false; dst[dstIdx++] = src[srcIdx++]; break; } if (dstIdx >= dstEnd) return false; dst[dstIdx++] = src[srcIdx++]; if ((src[srcIdx] & X86_MASK_JCC) != X86_INSTRUCTION_JCC) { // Not a relative jump if (src[srcIdx] == X86_ESCAPE) { srcIdx++; if (srcIdx >= codeEnd) return false; } if (dstIdx >= dstEnd) return false; dst[dstIdx++] = src[srcIdx++]; continue; } } else if ((src[srcIdx] & X86_MASK_JUMP) != X86_INSTRUCTION_JUMP) { // Not a relative call if (src[srcIdx] == X86_ESCAPE) { srcIdx++; if (srcIdx >= codeEnd) return false; } if (dstIdx >= dstEnd) return false; dst[dstIdx++] = src[srcIdx++]; continue; } if (srcIdx + 4 >= codeEnd) return false; if (dstIdx + 5 > dstEnd) return false; // Current instruction is a jump/call. Decode absolute address const int addr = BigEndian::readInt32(&src[srcIdx + 1]) ^ MASK_ADDRESS; const int64 offset = int64(addr) - int64(dstIdx); const int32 encodedOffset = (offset >= 0) ? int32(offset) : -int32((-offset) & X86_ADDR_MASK); dst[dstIdx++] = src[srcIdx++]; LittleEndian::writeInt32(&dst[dstIdx], encodedOffset); srcIdx += 4; dstIdx += 4; } if (dstIdx + (count - srcIdx) > dstEnd) return false; if (srcIdx < count) { memcpy(&dst[dstIdx], &src[srcIdx], size_t(count - srcIdx)); dstIdx += (count - srcIdx); } input._index += count; output._index += dstIdx; return true; } bool EXECodec::inverseARM(SliceArray& input, SliceArray& output, int count) { const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; int srcIdx = 9; int dstIdx = 0; const int dstEnd = output._length - output._index; const int codeStart = LittleEndian::readInt32(&src[1]); const int codeEnd = LittleEndian::readInt32(&src[5]); // Sanity check if ((codeStart < 0) || (codeEnd < srcIdx) || (codeEnd > count) || (codeStart > codeEnd - srcIdx) || (codeStart > dstEnd - dstIdx)) return false; if (codeStart > 0) { memcpy(&dst[dstIdx], &src[srcIdx], size_t(codeStart)); dstIdx += codeStart; srcIdx += codeStart; } while (srcIdx < codeEnd) { if (srcIdx + 4 > codeEnd) return false; if (dstIdx + 4 > dstEnd) return false; const int instr = LittleEndian::readInt32(&src[srcIdx]); const int opcode1 = instr & ARM_B_OPCODE_MASK; //const int opcode2 = instr & ARM_CB_OPCODE_MASK; bool isBL = (opcode1 == ARM_OPCODE_B) || (opcode1 == ARM_OPCODE_BL); // unconditional jump bool isCB = false; // disable for now ... isCB = (opcode2 == ARM_OPCODE_CBZ) || (opcode2 == ARM_OPCODE_CBNZ); // conditional jump if ((isBL == false) && (isCB == false)) { // Not a relative jump memcpy(&dst[dstIdx], &src[srcIdx], 4); srcIdx += 4; dstIdx += 4; continue; } // Decode absolute address int val, addr; if (isBL == true) { addr = (instr & ARM_B_ADDR_MASK) << 2; const int offset = (addr - dstIdx) >> 2; val = opcode1 | (offset & ARM_B_ADDR_MASK); } else { addr = ((instr & ARM_CB_ADDR_MASK) >> ARM_CB_REG_BITS) << 2; const int offset = (addr - dstIdx) >> 2; val = (instr & ~ARM_CB_ADDR_MASK) | (offset << ARM_CB_REG_BITS); } if (addr == 0) { if (srcIdx + 8 > codeEnd) return false; memcpy(&dst[dstIdx], &src[srcIdx + 4], 4); srcIdx += 8; dstIdx += 4; continue; } LittleEndian::writeInt32(&dst[dstIdx], val); srcIdx += 4; dstIdx += 4; } if (dstIdx + (count - srcIdx) > dstEnd) return false; if (srcIdx < count) { memcpy(&dst[dstIdx], &src[srcIdx], size_t(count - srcIdx)); dstIdx += (count - srcIdx); } input._index += count; output._index += dstIdx; return true; } kanzi::byte EXECodec::detectType(const kanzi::byte src[], int count, int& codeStart, int& codeEnd) { // Let us check the first bytes ... but this may not be the first block // Best effort const uint magic = Magic::getType(src); int arch = 0; const int blockSize = count; if (parseHeader(src, count, magic, arch, codeStart, codeEnd) == true) { if ((codeStart < 0) || (codeStart > blockSize) || (codeEnd < codeStart) || (codeEnd > blockSize)) return NOT_EXE | kanzi::byte(Global::UNDEFINED); switch(arch) { case ELF_X86_ARCH: case ELF_AMD64_ARCH: case WIN_X86_ARCH: case WIN_AMD64_ARCH: case MAC_AMD64_ARCH: return X86; case ELF_ARM64_ARCH: case WIN_ARM64_ARCH: case MAC_ARM64_ARCH: return ARM64; default: count = codeEnd - codeStart; } } if ((codeStart < 0) || (codeStart > blockSize) || (codeEnd < codeStart) || (codeEnd > blockSize)) return NOT_EXE | kanzi::byte(Global::UNDEFINED); if (count <= 0) return NOT_EXE | kanzi::byte(Global::UNDEFINED); int jumpsX86 = 0; int jumpsARM64 = 0; uint histo[256] = { 0 }; for (int i = codeStart; i < codeEnd; i++) { histo[int(src[i])]++; // X86 if (((i + 4) < codeEnd) && ((src[i] & X86_MASK_JUMP) == X86_INSTRUCTION_JUMP)) { if ((src[i + 4] == kanzi::byte(0)) || (src[i + 4] == kanzi::byte(0xFF))) { // Count relative jumps (CALL = E8/ JUMP = E9 .. .. .. 00/FF) jumpsX86++; continue; } } else if ((src[i] == X86_TWO_BYTE_PREFIX) && (i + 1 < codeEnd)) { int j = i + 1; if (((src[j] == kanzi::byte(0x38)) || (src[j] == kanzi::byte(0x3A))) && (j + 1 < codeEnd)) j++; // Count relative conditional jumps (0x0F 0x8?) with 16/32 offsets if ((src[j] & X86_MASK_JCC) == X86_INSTRUCTION_JCC) { jumpsX86++; i = j; continue; } i = j; } // ARM if (((i & 3) != 0) || (i + 4 > codeEnd)) continue; const int instr = LittleEndian::readInt32(&src[i]); const int opcode1 = instr & ARM_B_OPCODE_MASK; const int opcode2 = instr & ARM_CB_OPCODE_MASK; if ((opcode1 == ARM_OPCODE_B) || (opcode1 == ARM_OPCODE_BL) || (opcode2 == ARM_OPCODE_CBZ) || (opcode2 == ARM_OPCODE_CBNZ)) jumpsARM64++; } Global::DataType dt = Global::detectSimpleType(count, histo); if (dt != Global::BIN) return NOT_EXE | kanzi::byte(dt); // Filter out (some/many) multimedia files if ((histo[0] < uint(count / 10)) || (histo[255] < uint(count / 100))) return NOT_EXE | kanzi::byte(dt); int smallVals = 0; for (int i = 0; i < 16; i++) smallVals += histo[i]; if (smallVals > (count / 2)) return NOT_EXE | kanzi::byte(dt); // Ad-hoc thresholds if (jumpsX86 >= (count / 200)) return X86; if (jumpsARM64 >= (count / 200)) return ARM64; // Number of jump instructions too small => either not an exe or not worth the change, skip. return NOT_EXE | kanzi::byte(dt); } static bool setCodeRange(int count, int& codeStart, int& codeEnd, int64 start, int64 length) { if ((start < 0) || (length < 0) || (start > int64(count)) || (length > int64(count) - start)) return false; if (codeStart == 0) codeStart = int(start); codeEnd = int(start + length); return true; } // Return true if known header bool EXECodec::parseHeader(const kanzi::byte src[], int count, uint magic, int& arch, int& codeStart, int& codeEnd) { if (magic == Magic::WIN_MAGIC) { if (count >= 64) { const int posPE = LittleEndian::readInt32(&src[60]); if ((posPE > 0) && (posPE <= count - 48) && (LittleEndian::readInt32(&src[posPE]) == WIN_PE)) { const kanzi::byte* pe = &src[posPE]; if (setCodeRange(count, codeStart, codeEnd, int64(LittleEndian::readInt32(&pe[44])), int64(LittleEndian::readInt32(&pe[28]))) == false) return false; arch = LittleEndian::readInt16(&pe[4]); } return true; } } else if (magic == Magic::ELF_MAGIC) { bool isLittleEndian = src[5] == kanzi::byte(1); if (count >= 64) { codeStart = 0; if (isLittleEndian == true) { if (src[4] == kanzi::byte(2)) { // 64 bits const int nbEntries = int(LittleEndian::readInt16(&src[0x3C])); const int szEntry = int(LittleEndian::readInt16(&src[0x3A])); const int64 posSection = LittleEndian::readLong64(&src[0x28]); if ((szEntry <= 0) || (posSection < 0) || (posSection > int64(count) - 0x28)) return false; for (int i = 0; i < nbEntries; i++) { const int64 startEntry = posSection + int64(i) * int64(szEntry); if (startEntry > int64(count) - 0x28) return false; int typeSection = int(LittleEndian::readInt32(&src[startEntry + 4])); const int64 offSection = LittleEndian::readLong64(&src[startEntry + 0x18]); const int64 lenSection = LittleEndian::readLong64(&src[startEntry + 0x20]); if ((typeSection == 1) && (lenSection >= 64)) { if (setCodeRange(count, codeStart, codeEnd, offSection, lenSection) == false) return false; } } } else { // 32 bits const int nbEntries = int(LittleEndian::readInt16(&src[0x30])); const int szEntry = int(LittleEndian::readInt16(&src[0x2E])); const int64 posSection = int64(LittleEndian::readInt32(&src[0x20])); if ((szEntry <= 0) || (posSection < 0) || (posSection > int64(count) - 0x18)) return false; for (int i = 0; i < nbEntries; i++) { const int64 startEntry = posSection + int64(i) * int64(szEntry); if (startEntry > int64(count) - 0x18) return false; int typeSection = int(LittleEndian::readInt32(&src[startEntry + 4])); const int64 offSection = int64(LittleEndian::readInt32(&src[startEntry + 0x10])); const int64 lenSection = int64(LittleEndian::readInt32(&src[startEntry + 0x14])); if ((typeSection == 1) && (lenSection >= 64)) { if (setCodeRange(count, codeStart, codeEnd, offSection, lenSection) == false) return false; } } } arch = LittleEndian::readInt16(&src[18]); } else { if (src[4] == kanzi::byte(2)) { // 64 bits const int nbEntries = int(BigEndian::readInt16(&src[0x3C])); const int szEntry = int(BigEndian::readInt16(&src[0x3A])); const int64 posSection = BigEndian::readLong64(&src[0x28]); if ((szEntry <= 0) || (posSection < 0) || (posSection > int64(count) - 0x28)) return false; for (int i = 0; i < nbEntries; i++) { const int64 startEntry = posSection + int64(i) * int64(szEntry); if (startEntry > int64(count) - 0x28) return false; int typeSection = int(BigEndian::readInt32(&src[startEntry + 4])); const int64 offSection = BigEndian::readLong64(&src[startEntry + 0x18]); const int64 lenSection = BigEndian::readLong64(&src[startEntry + 0x20]); if ((typeSection == 1) && (lenSection >= 64)) { if (setCodeRange(count, codeStart, codeEnd, offSection, lenSection) == false) return false; } } } else { // 32 bits const int nbEntries = int(BigEndian::readInt16(&src[0x30])); const int szEntry = int(BigEndian::readInt16(&src[0x2E])); const int64 posSection = int64(BigEndian::readInt32(&src[0x20])); if ((szEntry <= 0) || (posSection < 0) || (posSection > int64(count) - 0x18)) return false; for (int i = 0; i < nbEntries; i++) { const int64 startEntry = posSection + int64(i) * int64(szEntry); if (startEntry > int64(count) - 0x18) return false; int typeSection = int(BigEndian::readInt32(&src[startEntry + 4])); const int64 offSection = int64(BigEndian::readInt32(&src[startEntry + 0x10])); const int64 lenSection = int64(BigEndian::readInt32(&src[startEntry + 0x14])); if ((typeSection == 1) && (lenSection >= 64)) { if (setCodeRange(count, codeStart, codeEnd, offSection, lenSection) == false) return false; } } } arch = BigEndian::readInt16(&src[18]); } codeStart = min(codeStart, count); codeEnd = min(codeEnd, count); return true; } } else if ((magic == Magic::MAC_MAGIC32) || (magic == Magic::MAC_CIGAM32) || (magic == Magic::MAC_MAGIC64) || (magic == Magic::MAC_CIGAM64)) { bool is64Bits = (magic == Magic::MAC_MAGIC64) || (magic == Magic::MAC_CIGAM64); codeStart = 0; static char MAC_TEXT_SEGMENT[] = "__TEXT"; static char MAC_TEXT_SECTION[] = "__text"; if (count >= 64) { int type = LittleEndian::readInt32(&src[12]); if (type != MAC_MH_EXECUTE) return false; arch = LittleEndian::readInt32(&src[4]); int nbCmds = LittleEndian::readInt32(&src[0x10]); int pos = (is64Bits == true) ? 0x20 : 0x1C; int cmd = 0; while (cmd < nbCmds) { if ((pos < 0) || (pos > count - 8)) return false; int ldCmd = LittleEndian::readInt32(&src[pos]); int szCmd = LittleEndian::readInt32(&src[pos + 4]); int szSegHdr = (is64Bits == true) ? 0x48 : 0x38; if ((szCmd < 8) || (szCmd > count - pos)) return false; if ((ldCmd == MAC_LC_SEGMENT) || (ldCmd == MAC_LC_SEGMENT64)) { if ((pos > count - 14) || (pos > count - szSegHdr)) return false; if (memcmp(&src[pos + 8], reinterpret_cast(MAC_TEXT_SEGMENT), 6) == 0) { int posSection = pos + szSegHdr; const int minSectionSize = (is64Bits == true) ? 0x38 : 0x30; if (posSection > count - minSectionSize) return false; if (memcmp(&src[posSection], reinterpret_cast(MAC_TEXT_SECTION), 6) == 0) { // Text section in TEXT segment if (is64Bits == true) { if (setCodeRange(count, codeStart, codeEnd, LittleEndian::readLong64(&src[posSection + 0x30]), int64(LittleEndian::readInt32(&src[posSection + 0x28]))) == false) return false; break; } else { if (setCodeRange(count, codeStart, codeEnd, int64(LittleEndian::readInt32(&src[posSection + 0x2C])), int64(LittleEndian::readInt32(&src[posSection + 0x28]))) == false) return false; break; } } } } cmd++; pos += szCmd; } codeStart = min(codeStart, count); codeEnd = min(codeEnd, count); return true; } } return false; } kanzi-cpp-2.6.0/src/transform/EXECodec.hpp000066400000000000000000000066731525576310500203320ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_EXECodec #define knz_EXECodec #include "../Context.hpp" #include "../Transform.hpp" namespace kanzi { class EXECodec FINAL : public Transform { public: EXECodec() { _pCtx = nullptr; } EXECodec(Context& ctx) : _pCtx(&ctx) {} ~EXECodec() {} bool forward(SliceArray& source, SliceArray& destination, int length); bool inverse(SliceArray& source, SliceArray& destination, int length); int getMaxEncodedLength(int inputLen) const; private: static const byte X86_MASK_JUMP; static const byte X86_INSTRUCTION_JUMP; static const byte X86_INSTRUCTION_JCC; static const byte X86_TWO_BYTE_PREFIX; static const byte X86_MASK_JCC; static const byte X86_ESCAPE; static const byte NOT_EXE; static const byte X86; static const byte ARM64; static const byte MASK_DT; static const int X86_ADDR_MASK; static const int MASK_ADDRESS; static const int ARM_B_ADDR_MASK; static const int ARM_B_OPCODE_MASK; static const int ARM_B_ADDR_SGN_MASK; static const int ARM_OPCODE_B; static const int ARM_OPCODE_BL; static const int ARM_CB_REG_BITS; static const int ARM_CB_ADDR_MASK; static const int ARM_CB_ADDR_SGN_MASK; static const int ARM_CB_OPCODE_MASK; static const int ARM_OPCODE_CBZ; static const int ARM_OPCODE_CBNZ; static const int WIN_PE; static const uint16 WIN_X86_ARCH; static const uint16 WIN_AMD64_ARCH; static const uint16 WIN_ARM64_ARCH; static const int ELF_X86_ARCH; static const int ELF_AMD64_ARCH; static const int ELF_ARM64_ARCH; static const int MAC_AMD64_ARCH; static const int MAC_ARM64_ARCH; static const int MAC_MH_EXECUTE; static const int MAC_LC_SEGMENT; static const int MAC_LC_SEGMENT64; static const int MIN_BLOCK_SIZE; static const int MAX_BLOCK_SIZE; bool forwardARM(SliceArray& source, SliceArray& destination, int length, int codeStart, int codeEnd); bool forwardX86(SliceArray& source, SliceArray& destination, int length, int codeStart, int codeEnd); bool inverseARM(SliceArray& source, SliceArray& destination, int length); bool inverseX86(SliceArray& source, SliceArray& destination, int length); static byte detectType(const byte src[], int count, int& codeStart, int& codeEnd); static bool parseHeader(const byte src[], int count, uint magic, int& arch, int& codeStart, int& codeEnd); Context* _pCtx; }; inline int EXECodec::getMaxEncodedLength(int srcLen) const { // Allocate some extra buffer for incompressible data. return (srcLen <= 256) ? srcLen + 32 : srcLen + srcLen / 8; } } #endif kanzi-cpp-2.6.0/src/transform/FSDCodec.cpp000066400000000000000000000343261525576310500203140ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include "FSDCodec.hpp" #include "../Global.hpp" #include "../Magic.hpp" #include "../Memory.hpp" using namespace kanzi; using namespace std; const int FSDCodec::MIN_LENGTH = 1024; const kanzi::byte FSDCodec::ESCAPE_TOKEN = kanzi::byte(255); const kanzi::byte FSDCodec::DELTA_CODING = kanzi::byte(0); const kanzi::byte FSDCodec::XOR_CODING = kanzi::byte(1); const int8 FSDCodec::ZIGZAG2[256] = { 0, -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7, -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, -14, 14, -15, 15, -16, 16, -17, 17, -18, 18, -19, 19, -20, 20, -21, 21, -22, 22, -23, 23, -24, 24, -25, 25, -26, 26, -27, 27, -28, 28, -29, 29, -30, 30, -31, 31, -32, 32, -33, 33, -34, 34, -35, 35, -36, 36, -37, 37, -38, 38, -39, 39, -40, 40, -41, 41, -42, 42, -43, 43, -44, 44, -45, 45, -46, 46, -47, 47, -48, 48, -49, 49, -50, 50, -51, 51, -52, 52, -53, 53, -54, 54, -55, 55, -56, 56, -57, 57, -58, 58, -59, 59, -60, 60, -61, 61, -62, 62, -63, 63, -64, 64, -65, 65, -66, 66, -67, 67, -68, 68, -69, 69, -70, 70, -71, 71, -72, 72, -73, 73, -74, 74, -75, 75, -76, 76, -77, 77, -78, 78, -79, 79, -80, 80, -81, 81, -82, 82, -83, 83, -84, 84, -85, 85, -86, 86, -87, 87, -88, 88, -89, 89, -90, 90, -91, 91, -92, 92, -93, 93, -94, 94, -95, 95, -96, 96, -97, 97, -98, 98, -99, 99, -100, 100, -101, 101, -102, 102, -103, 103, -104, 104, -105, 105, -106, 106, -107, 107, -108, 108, -109, 109, -110, 110, -111, 111, -112, 112, -113, 113, -114, 114, -115, 115, -116, 116, -117, 117, -118, 118, -119, 119, -120, 120, -121, 121, -122, 122, -123, 123, -124, 124, -125, 125, -126, 126, -127, 127, -128, }; bool FSDCodec::forward(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (!SliceArray::isValid(input)) throw invalid_argument("FSD codec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("FSD codec: Invalid output block"); if (input._array == output._array) return false; if (output._length - output._index < getMaxEncodedLength(count)) return false; // If too small, skip if (count < MIN_LENGTH) return false; if (_pCtx != nullptr) { Global::DataType dt = (Global::DataType) _pCtx->getInt("dataType", Global::UNDEFINED); if ((dt != Global::UNDEFINED) && (dt != Global::MULTIMEDIA) && (dt != Global::BIN)) return false; } const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; uint magic = Magic::getType(src); // Skip detection except for a few candidate types switch (magic) { case Magic::BMP_MAGIC: case Magic::RIFF_MAGIC: case Magic::PBM_MAGIC: case Magic::PGM_MAGIC: case Magic::PPM_MAGIC: case Magic::NO_MAGIC: break; default: return false; } const int srcEnd = count; const int count10 = count / 10; const int count5 = 2 * count10; // count5=count/5 does not guarantee count5=2*count10 ! uint histo[7][256]; memset(&histo[0][0], 0, sizeof(histo)); // Check several step values on a few sub-blocks (no memory allocation) const kanzi::byte* in0 = &src[count5 * 0]; const kanzi::byte* in1 = &src[count5 * 2]; const kanzi::byte* in2 = &src[count5 * 4]; for (int i = count10; i < count5; i++) { const kanzi::byte b0 = in0[i]; histo[0][int(b0)]++; histo[1][int(b0 ^ in0[i - 1])]++; histo[2][int(b0 ^ in0[i - 2])]++; histo[3][int(b0 ^ in0[i - 3])]++; histo[4][int(b0 ^ in0[i - 4])]++; histo[5][int(b0 ^ in0[i - 8])]++; histo[6][int(b0 ^ in0[i - 16])]++; const kanzi::byte b1 = in1[i]; histo[0][int(b1)]++; histo[1][int(b1 ^ in1[i - 1])]++; histo[2][int(b1 ^ in1[i - 2])]++; histo[3][int(b1 ^ in1[i - 3])]++; histo[4][int(b1 ^ in1[i - 4])]++; histo[5][int(b1 ^ in1[i - 8])]++; histo[6][int(b1 ^ in1[i - 16])]++; const kanzi::byte b2 = in2[i]; histo[0][int(b2)]++; histo[1][int(b2 ^ in2[i - 1])]++; histo[2][int(b2 ^ in2[i - 2])]++; histo[3][int(b2 ^ in2[i - 3])]++; histo[4][int(b2 ^ in2[i - 4])]++; histo[5][int(b2 ^ in2[i - 8])]++; histo[6][int(b2 ^ in2[i - 16])]++; } // Find if entropy is lower post transform int minIdx = 0; int ent[7]; for (int i = 0; i < 7; i++) { ent[i] = Global::computeOrder0Entropy1024(3 * count10, histo[i]); if (ent[i] < ent[minIdx]) minIdx = i; } // If not better, quick exit if (ent[minIdx] >= ent[0]) { if (_pCtx != nullptr) _pCtx->putInt("dataType", Global::detectSimpleType(3 * count10, histo[0])); return false; } if (_pCtx != nullptr) _pCtx->putInt("dataType", Global::MULTIMEDIA); const int distances[7] = { 0, 1, 2, 3, 4, 8, 16 }; const int dist = distances[minIdx]; const kanzi::byte coding = DELTA_CODING; // Keep triplet-correlated data interleaved since phase bucketing can // disrupt downstream matches for this layout. const bool bucketed = (dist > 1) && (dist != 3) && (dist != 16); const kanzi::byte mode = kanzi::byte(int(coding) | (bucketed == true ? 2 : 0)); dst[0] = mode; dst[1] = kanzi::byte(dist); int srcIdx = 0; int dstIdx = 2; // Emit first bytes for (int i = 0; i < dist; i++) dst[dstIdx++] = src[srcIdx++]; // Emit modified bytes. The bucketed layout keeps each phase together. if (bucketed == true) { const int bucketLength = 1 << 15; const int tileLength = dist * bucketLength; for (int tileStart = 0; tileStart < srcEnd; tileStart += tileLength) { const int tileEnd = min(tileStart + tileLength, srcEnd); for (int lane = 0; lane < dist; lane++) { const int firstPos = tileStart + lane + ((tileStart == 0) ? dist : 0); for (int pos = firstPos; pos < tileEnd; pos += dist) { const uint residual = uint(uint8(int(src[pos]) - int(src[pos - dist]))); const uint zigzag = (residual & 0x80) ? ((256 - residual) << 1) - 1 : residual << 1; dst[dstIdx++] = kanzi::byte(zigzag); } } } srcIdx = srcEnd; } else { while (srcIdx < srcEnd) { // Encode the delta modulo 256. The signed difference is not // needed to reconstruct a byte, and all 256 residuals fit in // one byte. Values in [-127..127] retain the previous zigzag // mapping; -128 and +128 share the same modular residual. const uint residual = uint(uint8(int(src[srcIdx]) - int(src[srcIdx - dist]))); const uint zigzag = (residual & 0x80) ? ((256 - residual) << 1) - 1 : residual << 1; dst[dstIdx++] = kanzi::byte(zigzag); srcIdx++; } } if (srcIdx != srcEnd) return false; // Extra check that the transform makes sense memset(&histo[0][0], 0, sizeof(uint) * 256); const kanzi::byte* out1 = &dst[count5 * 1]; const kanzi::byte* out2 = &dst[count5 * 3]; for (int i = 0; i < count10; i++) { histo[0][int(out1[i])]++; histo[0][int(out2[i])]++; } const int entropy = Global::computeOrder0Entropy1024(count5, histo[0]); if (entropy >= ent[0]) return false; input._index += srcIdx; output._index += dstIdx; return true; // Allowed to expand } bool FSDCodec::inverse(SliceArray& input, SliceArray& output, int count) { const int bsVersion = (_pCtx == nullptr) ? 7 : _pCtx->getInt("bsVersion", 7); if (count == 0) return true; if (!SliceArray::isValid(input)) throw invalid_argument("FSD codec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("FSD codec: Invalid output block"); if (input._array == output._array) return false; if (count < 4) return false; if (input._index + count > input._length) return false; const int srcEnd = count; const int dstEnd = output._length - output._index; const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; // Retrieve mode & step value const kanzi::byte mode = src[0]; const int modeValue = int(mode); const kanzi::byte coding = kanzi::byte(modeValue & 1); const bool bucketed = ((modeValue & 2) != 0); const int dist = int(src[1]); // Sanity check if ((dist < 1) || ((dist > 4) && (dist != 8) && (dist != 16))) return false; if ((bsVersion >= 7) && ((modeValue & ~3) != 0)) return false; const int dataLength = count - 2; if ((count < dist + 2) || (dist > dstEnd) || (bucketed && (dataLength > dstEnd))) return false; // Emit first bytes memcpy(&dst[0], &src[2], size_t(dist)); int srcIdx = dist + 2; int dstIdx = dist; // Recover original bytes if (bsVersion < 7) { if (mode == DELTA_CODING) { while ((srcIdx < srcEnd) && (dstIdx < dstEnd)) { if (src[srcIdx] != ESCAPE_TOKEN) { const int value = int(src[srcIdx]); const int delta = (value >> 1) ^ -(value & 1); dst[dstIdx] = kanzi::byte(int(dst[dstIdx - dist]) + delta); srcIdx++; dstIdx++; continue; } srcIdx++; if (srcIdx == srcEnd) return false; dst[dstIdx] = src[srcIdx] ^ dst[dstIdx - dist]; srcIdx++; dstIdx++; } } else if (mode == XOR_CODING) { if (dist == 16) { while ((srcIdx + 16 <= srcEnd) && (dstIdx + 16 <= dstEnd)) { KANZI_MEM_XOR16(&dst[dstIdx], &src[srcIdx], &dst[dstIdx - 16]); srcIdx += 16; dstIdx += 16; } } else if (dist == 8) { while ((srcIdx + 8 <= srcEnd) && (dstIdx + 8 <= dstEnd)) { KANZI_MEM_XOR8(&dst[dstIdx], &src[srcIdx], &dst[dstIdx - 8]); srcIdx += 8; dstIdx += 8; } } while ((srcIdx < srcEnd) && (dstIdx < dstEnd)) { dst[dstIdx] = src[srcIdx] ^ dst[dstIdx - dist]; srcIdx++; dstIdx++; } } else { // Invalid mode return false; } } else if (bucketed) { const int bucketLength = 1 << 15; const int tileLength = dist * bucketLength; for (int tileStart = 0; tileStart < dataLength; tileStart += tileLength) { const int tileEnd = min(tileStart + tileLength, dataLength); for (int lane = 0; lane < dist; lane++) { const int firstPos = tileStart + lane + ((tileStart == 0) ? dist : 0); for (int pos = firstPos; pos < tileEnd; pos += dist) { if (srcIdx >= srcEnd) return false; if (coding == DELTA_CODING) { const uint value = uint(uint8(src[srcIdx++])); dst[pos] = kanzi::byte(int(dst[pos - dist]) + int(ZIGZAG2[value])); } else { dst[pos] = src[srcIdx++] ^ dst[pos - dist]; } } } } dstIdx = dataLength; } else if (coding == DELTA_CODING) { while ((srcIdx < srcEnd) && (dstIdx < dstEnd)) { const uint value = uint(uint8(src[srcIdx++])); const int delta = int(ZIGZAG2[value]); dst[dstIdx] = kanzi::byte(int(dst[dstIdx - dist]) + delta); dstIdx++; } } else if (coding == XOR_CODING) { if (dist == 16) { while ((srcIdx + 16 <= srcEnd) && (dstIdx + 16 <= dstEnd)) { KANZI_MEM_XOR16(&dst[dstIdx], &src[srcIdx], &dst[dstIdx - 16]); srcIdx += 16; dstIdx += 16; } } else if (dist == 8) { while ((srcIdx + 8 <= srcEnd) && (dstIdx + 8 <= dstEnd)) { KANZI_MEM_XOR8(&dst[dstIdx], &src[srcIdx], &dst[dstIdx - 8]); srcIdx += 8; dstIdx += 8; } } while ((srcIdx < srcEnd) && (dstIdx < dstEnd)) { dst[dstIdx] = src[srcIdx] ^ dst[dstIdx - dist]; srcIdx++; dstIdx++; } } else { // Invalid mode return false; } input._index += srcIdx; output._index += dstIdx; return srcIdx == srcEnd; } kanzi-cpp-2.6.0/src/transform/FSDCodec.hpp000066400000000000000000000030731525576310500203140ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_FSDCodec #define knz_FSDCodec #include "../Context.hpp" #include "../Transform.hpp" // Fixed Step Delta codec // Decorrelate values separated by a constant distance (step) and encode residuals namespace kanzi { class FSDCodec FINAL : public Transform { public: FSDCodec() { _pCtx = nullptr; } FSDCodec(Context& ctx) : _pCtx(&ctx) {} ~FSDCodec() {} bool forward(SliceArray& src, SliceArray& dst, int length); bool inverse(SliceArray& src, SliceArray& dst, int length); // Required encoding output buffer size int getMaxEncodedLength(int srcLen) const { return srcLen + ((srcLen < 1024) ? 64 : srcLen >> 4); // limit expansion } private: static const int MIN_LENGTH; static const byte ESCAPE_TOKEN; static const byte DELTA_CODING; static const byte XOR_CODING; static const int8 ZIGZAG2[256]; Context* _pCtx; }; } #endif kanzi-cpp-2.6.0/src/transform/LZCodec.cpp000066400000000000000000001116541525576310500202250ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include "LZCodec.hpp" #include "../Memory.hpp" #include "TransformFactory.hpp" using namespace kanzi; using namespace std; LZCodec::LZCodec() { _delegate = new LZXCodec(); } LZCodec::LZCodec(Context& ctx) { const int lzType = ctx.getInt("lz", TransformFactory::LZ_TYPE); if (lzType == TransformFactory::LZP_TYPE) { _delegate = new LZPCodec(ctx); } else if (lzType == TransformFactory::LZX_TYPE) { _delegate = new LZXCodec(ctx); } else { _delegate = new LZXCodec(ctx); } } bool LZCodec::forward(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (input._array == output._array) return false; return _delegate->forward(input, output, count); } bool LZCodec::inverse(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (input._array == output._array) return false; return _delegate->inverse(input, output, count); } template<> const uint LZXCodec::HASH_SEED = 0x1E35A7BD; template<> const uint LZXCodec::HASH_LOG = 16; template<> const uint LZXCodec::HASH_RSHIFT = 64 - HASH_LOG; template<> const uint LZXCodec::HASH_LSHIFT = 24; template<> const int LZXCodec::MAX_DISTANCE1 = (1 << 16) - 2; template<> const int LZXCodec::MAX_DISTANCE2 = (1 << 24) - 2; template<> const int LZXCodec::MIN_MATCH4 = 4; template<> const int LZXCodec::MIN_MATCH6 = 6; template<> const int LZXCodec::MIN_MATCH9 = 9; template<> const int LZXCodec::MAX_MATCH = 65535 + 254 + MIN_MATCH4; template<> const int LZXCodec::MIN_BLOCK_LENGTH = 24; template<> const int LZXCodec::READ_LENGTH_GUARD = 4; template<> const uint LZXCodec::HASH_SEED = 0x1E35A7BD; template<> const uint LZXCodec::HASH_LOG = 19; template<> const uint LZXCodec::HASH_RSHIFT = 64 - HASH_LOG; template<> const uint LZXCodec::HASH_LSHIFT = 24; template<> const int LZXCodec::MAX_DISTANCE1 = (1 << 16) - 2; template<> const int LZXCodec::MAX_DISTANCE2 = (1 << 24) - 2; template<> const int LZXCodec::MIN_MATCH4 = 4; template<> const int LZXCodec::MIN_MATCH6 = 6; template<> const int LZXCodec::MIN_MATCH9 = 9; template<> const int LZXCodec::MAX_MATCH = 65535 + 254 + MIN_MATCH4; template<> const int LZXCodec::MIN_BLOCK_LENGTH = 24; template<> const int LZXCodec::READ_LENGTH_GUARD = 4; template bool LZXCodec::forward(SliceArray& input, SliceArray& output, int count) { // Forward LZ processing: // 1. Allocate/reset the hash table and staging buffers. // 2. Reserve the last 18 input bytes as a safe literal tail. // 3. At each position: // - hash the current position; // - test the two repeat distances at srcIdx + 1; // - if no repeat match is found, test the latest hash candidate at srcIdx; // - optionally test matches at srcIdx + 1 and srcIdx + 2; // - extend the selected match backwards to reduce the literal run. // 4. Emit the literal length, repeat or explicit distance, match length, // and literals. // 5. Insert all positions covered by the match into the hash table. // 6. Append the final literals and concatenate the staging buffers. if (count == 0) return true; if (!SliceArray::isValid(input)) throw invalid_argument("LZ codec: Invalid input block"); if (count > input._length - input._index) return false; if (!SliceArray::isValid(output)) throw invalid_argument("LZ codec: Invalid output block"); if (output._length - output._index < getMaxEncodedLength(count)) return false; // If too small, skip if (count < MIN_BLOCK_LENGTH) return false; if (_hashSize == 0) { const int newSize = 1 << HASH_LOG; int32* hashes = new int32[newSize]; delete[] _hashes; _hashes = hashes; _hashSize = newSize; } if (_bufferSize < max(count / 5, 256)) { const int newSize = max(count / 5, 256); kanzi::byte* mLenBuf = new kanzi::byte[newSize]; kanzi::byte* mBuf = nullptr; kanzi::byte* tkBuf = nullptr; try { mBuf = new kanzi::byte[newSize]; tkBuf = new kanzi::byte[newSize]; } catch (...) { delete[] mLenBuf; delete[] mBuf; delete[] tkBuf; throw; } delete[] _mLenBuf; delete[] _mBuf; delete[] _tkBuf; _mLenBuf = mLenBuf; _mBuf = mBuf; _tkBuf = tkBuf; _bufferSize = newSize; } memset(_hashes, 0, sizeof(int32) * _hashSize); const int srcEnd = count - 16 - 2; const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; const int maxDist = (srcEnd < 4 * MAX_DISTANCE1) ? MAX_DISTANCE1 : MAX_DISTANCE2; dst[12] = (maxDist == MAX_DISTANCE1) ? kanzi::byte(0) : kanzi::byte(1); int mm = MIN_MATCH4; if (_pCtx != nullptr) { Global::DataType dt = (Global::DataType)_pCtx->getInt("dataType", Global::UNDEFINED); if (dt == Global::DNA) { // Longer min match for DNA input mm = MIN_MATCH6; } else if (dt == Global::SMALL_ALPHABET) { return false; } } // dst[12] = 0000MMMD (4 bits + 3 bits minMatch + 1 bit max distance) dst[12] |= kanzi::byte(((mm - 2) & 0x07) << 1); // minMatch in [2..9] const int minMatch = mm; int srcIdx = 0; int dstIdx = 13; int anchor = 0; int mIdx = 0; int mLenIdx = 0; int tkIdx = 0; int repd[] = { count, count }; int repIdx = 0; int srcInc = 0; while (srcIdx < srcEnd) { int bestLen = 0; const int32 h0 = hash(&src[srcIdx]); const int ref0 = _hashes[h0]; _hashes[h0] = srcIdx; const int srcIdx1 = srcIdx + 1; int ref = srcIdx1 - repd[repIdx]; const int minRef = max(srcIdx - maxDist, 0); if ((ref > minRef) && KANZI_MEM_EQ4(&src[srcIdx1], &src[ref])) { // Check repd0 first bestLen = findMatch(src, srcIdx1, ref, min(srcEnd - srcIdx1, MAX_MATCH)); } else { ref = srcIdx1 - repd[repIdx ^ 1]; if ((ref > minRef) && KANZI_MEM_EQ4(&src[srcIdx1], &src[ref])) { // Check repd1 first bestLen = findMatch(src, srcIdx1, ref, min(srcEnd - srcIdx1, MAX_MATCH)); } } if (bestLen < minMatch) { // Check match at position in hash table ref = ref0; if ((ref > minRef) && KANZI_MEM_EQ4(&src[srcIdx], &src[ref])) { bestLen = findMatch(src, srcIdx, ref, min(srcEnd - srcIdx, MAX_MATCH)); } // No good match ? if (bestLen < minMatch) { srcIdx = srcIdx1 + (srcInc >> 6); srcInc++; repIdx = 0; continue; } if ((srcIdx - ref != repd[0]) && (srcIdx - ref != repd[1])) { // Check if better match at next position const int32 h1 = hash(&src[srcIdx1]); const int ref1 = _hashes[h1]; _hashes[h1] = srcIdx1; if ((ref1 > minRef + 1) && KANZI_MEM_EQ4(&src[srcIdx1 + bestLen - 3], &src[ref1 + bestLen - 3])) { const int bestLen1 = findMatch(src, srcIdx1, ref1, min(srcEnd - srcIdx1, MAX_MATCH)); // Select best match if (bestLen1 >= bestLen) { ref = ref1; bestLen = bestLen1; srcIdx = srcIdx1; } } if (T == true) { const int srcIdx2 = srcIdx1 + 1; const int32 h2 = hash(&src[srcIdx2]); const int ref2 = _hashes[h2]; _hashes[h2] = srcIdx2; if ((ref2 > minRef + 2) && KANZI_MEM_EQ4(&src[srcIdx2 + bestLen - 3], &src[ref2 + bestLen - 3])) { const int bestLen2 = findMatch(src, srcIdx2, ref2, min(srcEnd - srcIdx2, MAX_MATCH)); // Select best match if (bestLen2 >= bestLen) { ref = ref2; bestLen = bestLen2; srcIdx = srcIdx2; } } } } // Extend backwards while ((srcIdx > anchor) && (ref > minRef) && (src[srcIdx - 1] == src[ref - 1])) { bestLen++; ref--; srcIdx--; } if (bestLen > MAX_MATCH) { ref += (bestLen - MAX_MATCH); srcIdx += (bestLen - MAX_MATCH); bestLen = MAX_MATCH; } } else { if ((bestLen >= MAX_MATCH) || (src[srcIdx] != src[ref - 1])) { srcIdx++; const int32 h1 = hash(&src[srcIdx]); _hashes[h1] = srcIdx; } else { bestLen++; ref--; } } // Emit match srcInc = 0; // Token: 3 bits litLen + 2 bits flag + 3 bits mLen (LLLFFMMM) // or 3 bits litLen + 3 bits flag + 2 bits mLen (LLLFFFMM) // LLL : <= 7 --> LLL == literal length (if 7, remainder encoded outside of token) // MMM : <= 7 --> MMM == match length (if 7, remainder encoded outside of token) // MM : <= 3 --> MM == match length (if 3, remainder encoded outside of token) // FF = 01 --> 1 byte dist // FF = 10 --> 2 byte dist // FF = 11 --> 3 byte dist // FFF = 000 --> dist == repd0 // FFF = 001 --> dist == repd1 const int dist = srcIdx - ref; int token, mLenTh; if (dist == repd[0]) { token = 0x00; mLenTh = 3; } else if (dist == repd[1]) { token = 0x04; mLenTh = 3; } else { // Emit distance (since not repeat) // 1-byte distance: dist - 1 // 2-byte distance: dist - 257 // 3-byte distance: dist - 65793 const int encodedDist = dist - 1; if (encodedDist < 256) { _mBuf[mIdx++] = kanzi::byte(encodedDist); token = 0x08; } else if (encodedDist < 65792) { const uint32 value = uint32(encodedDist - 256) << 16; BigEndian::writeInt32(&_mBuf[mIdx], int32(value)); mIdx += 2; token = 0x10; } else { const uint32 value = uint32(encodedDist - 65792) << 8; BigEndian::writeInt32(&_mBuf[mIdx], int32(value)); mIdx += 3; token = 0x18; } mLenTh = 7; } const int mLen = bestLen - minMatch; // Emit match length if (mLen >= mLenTh) { token += mLenTh; mLenIdx += emitLength(&_mLenBuf[mLenIdx], mLen - mLenTh); } else { token += mLen; } repd[1] = repd[0]; repd[0] = dist; repIdx = 1; const int litLen = srcIdx - anchor; // Emit token // Literals to process ? if (litLen == 0) { _tkBuf[tkIdx++] = kanzi::byte(token); } else { // Emit literal length if (litLen >= 7) { if (litLen >= (1 << 24)) return false; _tkBuf[tkIdx++] = kanzi::byte((7 << 5) | token); dstIdx += emitLength(&dst[dstIdx], litLen - 7); } else { _tkBuf[tkIdx++] = kanzi::byte((litLen << 5) | token); } // Emit literals emitLiterals(&src[anchor], &dst[dstIdx], litLen); dstIdx += litLen; } if (mIdx >= _bufferSize - 8) { // Expand match buffer kanzi::byte* mBuf = new kanzi::byte[(_bufferSize * 3) / 2]; memcpy(&mBuf[0], &_mBuf[0], _bufferSize); if ( _mBuf != nullptr) delete[] _mBuf; _mBuf = mBuf; if (mLenIdx >= _bufferSize - 8) { kanzi::byte* mLenBuf = new kanzi::byte[(_bufferSize * 3) / 2]; memcpy(&mLenBuf[0], &_mLenBuf[0], _bufferSize); if (_mLenBuf != nullptr) delete[] _mLenBuf; _mLenBuf = mLenBuf; } _bufferSize = (_bufferSize * 3) / 2; } // Fill _hashes and update positions. The current position was already // inserted above; sample four positions out of every eight (offsets // 0, 3, 5, and 6; two even and two odd) to reduce hash-table work // while retaining coverage. anchor = srcIdx + bestLen; int hashIdx = srcIdx + 1; while (hashIdx + 6 < anchor) { const int32 hh0 = hash(&src[hashIdx]); const int32 hh1 = hash(&src[hashIdx + 3]); const int32 hh2 = hash(&src[hashIdx + 5]); const int32 hh3 = hash(&src[hashIdx + 6]); _hashes[hh0] = hashIdx; _hashes[hh1] = hashIdx + 3; _hashes[hh2] = hashIdx + 5; _hashes[hh3] = hashIdx + 6; hashIdx += 8; } while (hashIdx < anchor) { const int32 h = hash(&src[hashIdx]); _hashes[h] = hashIdx; hashIdx++; } srcIdx = anchor; } // Emit last literals const int litLen = count - anchor; if (dstIdx + litLen + tkIdx + mIdx + mLenIdx >= count) return false; if (litLen >= 7) { _tkBuf[tkIdx++] = kanzi::byte(7 << 5); dstIdx += emitLength(&dst[dstIdx], litLen - 7); } else { _tkBuf[tkIdx++] = kanzi::byte(litLen << 5); } memcpy(&dst[dstIdx], &src[anchor], litLen); dstIdx += litLen; // Emit buffers: literals + tokens + matches LittleEndian::writeInt32(&dst[0], dstIdx); LittleEndian::writeInt32(&dst[4], tkIdx); LittleEndian::writeInt32(&dst[8], mIdx); memcpy(&dst[dstIdx], &_tkBuf[0], tkIdx); dstIdx += tkIdx; memcpy(&dst[dstIdx], &_mBuf[0], mIdx); dstIdx += mIdx; memcpy(&dst[dstIdx], &_mLenBuf[0], mLenIdx); dstIdx += mLenIdx; input._index += count; output._index += dstIdx; return dstIdx <= count - (count / 100); } template bool LZXCodec::inverse(SliceArray& input, SliceArray& output, int count) { // CompressedInputStream supplies trailing input padding. The versioned // decoders rely on READ_LENGTH_GUARD bytes for readLength() lookahead. int bsVersion = _pCtx == nullptr ? 6 : _pCtx->getInt("bsVersion", 6); if (bsVersion < 6) return inverseV5(input, output, count); if (bsVersion < 7) return inverseV6(input, output, count); return inverseV7(input, output, count); } template bool LZXCodec::inverseV6(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (count < 13) return false; if (!SliceArray::isValid(input)) throw invalid_argument("LZ codec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("LZ codec: Invalid output block"); const int inputSize = input._length - input._index; // readLength() reads four bytes for a three-byte length encoding. if ((inputSize < READ_LENGTH_GUARD) || (count > inputSize - READ_LENGTH_GUARD)) return false; const int dstCapacity = output._length - output._index; // A 16-byte match copy may write up to 15 bytes past the logical match end. if (dstCapacity < 15) return false; const int dstEnd = dstCapacity - 15; kanzi::byte* dst = &output._array[output._index]; const kanzi::byte* src = &input._array[input._index]; int tkIdx = LittleEndian::readInt32(&src[0]); int mIdx = LittleEndian::readInt32(&src[4]); int mLenIdx = LittleEndian::readInt32(&src[8]); // Sanity checks if ((tkIdx < 0) || (mIdx < 0) || (mLenIdx < 0)) return false; if ((tkIdx <= 13) || (tkIdx > count) || (mIdx > count - tkIdx) || (mLenIdx > count - tkIdx - mIdx)) return false; mIdx += tkIdx; mLenIdx += mIdx; const int tokenEnd = mIdx; const int matchEnd = mLenIdx; const int srcEnd = tkIdx - 13; const int litEnd = tkIdx; const int maxDist = ((int(src[12]) & 1) == 0) ? MAX_DISTANCE1 : MAX_DISTANCE2; const int minMatch = ((int(src[12]) >> 1) & 0x07) + 2; bool res = true; int srcIdx = 13; int dstIdx = 0; int repd0 = count; int repd1 = count; while (true) { const int token = int(src[tkIdx++]); // Get match length and distance int mLen, dist; if ((token & 0x18) == 0) { // Repetition distance, read mLen remainder (if any) outside of token mLen = token & 0x03; if (KANZI_UNLIKELY((mLen == 3) && (mLenIdx >= count))) { res = false; goto exit; } mLen += (mLen == 3 ? minMatch + int(readLength(src, mLenIdx)) : minMatch); dist = (token & 0x04) == 0 ? repd0 : repd1; } else { // Read mLen remainder (if any) outside of token mLen = token & 0x07; if (KANZI_UNLIKELY((mLen == 7) && (mLenIdx >= count))) { res = false; goto exit; } mLen += (mLen == 7 ? minMatch + int(readLength(src, mLenIdx)) : minMatch); if (KANZI_UNLIKELY(mIdx >= count)) { res = false; goto exit; } dist = int(src[mIdx++]); const int f1 = (token >> 4) & 1; const int f2 = (token >> 3) & f1; dist = (dist << (8 & -f1)) | (-f1 & int(src[mIdx])); mIdx += f1; dist = (dist << (8 & -f2)) | (-f2 & int(src[mIdx])); mIdx += f2; } if (token >= 32) { // Get literal length const uint litLen = (token >= 0xE0) ? uint(7) + readLength(src, srcIdx) : uint(token >> 5); if ((litLen > uint(dstEnd - dstIdx)) || (litLen > uint(litEnd - srcIdx))) { res = false; goto exit; } // Emit literals const kanzi::byte* s = &src[srcIdx]; kanzi::byte* d = &dst[dstIdx]; srcIdx += int(litLen); dstIdx += int(litLen); if (srcIdx >= srcEnd) { memcpy(d, s, litLen); break; } emitLiterals(s, d, litLen); } repd1 = repd0; repd0 = dist; const int mEnd = dstIdx + mLen; int ref = dstIdx - dist; // Sanity check if ((dist == 0) || (ref < 0) || (dist > maxDist) || (mEnd > dstEnd)) { res = false; goto exit; } if ((dist >= 64) && (mLen >= 64)) prefetchRead(&dst[dstIdx + 64]); // Copy match if (dist >= 16) { do { // The stream decoder supplies trailing padding for this // 16-byte copy, which may write up to 15 bytes past mEnd. KANZI_MEM_CP16(&dst[dstIdx], &dst[ref]); ref += 16; dstIdx += 16; } while (dstIdx < mEnd); } else if (dist != 1) { const kanzi::byte* s = &dst[ref]; kanzi::byte* p = &dst[dstIdx]; const kanzi::byte* pend = &p[mLen]; while (p < pend) *p++ = *s++; } else { // dist = 1 memset(&dst[dstIdx], int(dst[ref]), mLen); } dstIdx = mEnd; } exit: output._index += dstIdx; input._index += count; return res && (srcIdx == srcEnd + 13) && (tkIdx == tokenEnd) && (mIdx == matchEnd) && (mLenIdx == count); } template bool LZXCodec::inverseV7(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (count < 13) return false; if (!SliceArray::isValid(input)) throw invalid_argument("LZ codec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("LZ codec: Invalid output block"); const int inputSize = input._length - input._index; // readLength() reads four bytes for a three-byte length encoding. if ((inputSize < READ_LENGTH_GUARD) || (count > inputSize - READ_LENGTH_GUARD)) return false; const int dstCapacity = output._length - output._index; // A 16-byte match copy may write up to 15 bytes past the logical match end. if (dstCapacity < 15) return false; const int dstEnd = dstCapacity - 15; kanzi::byte* dst = &output._array[output._index]; const kanzi::byte* src = &input._array[input._index]; int tkIdx = LittleEndian::readInt32(&src[0]); int mIdx = LittleEndian::readInt32(&src[4]); int mLenIdx = LittleEndian::readInt32(&src[8]); // Sanity checks if ((tkIdx < 0) || (mIdx < 0) || (mLenIdx < 0)) return false; if ((tkIdx <= 13) || (tkIdx > count) || (mIdx > count - tkIdx) || (mLenIdx > count - tkIdx - mIdx)) return false; mIdx += tkIdx; mLenIdx += mIdx; const int tokenEnd = mIdx; const int matchEnd = mLenIdx; const int srcEnd = tkIdx - 13; const int litEnd = tkIdx; const int maxDist = ((int(src[12]) & 1) == 0) ? MAX_DISTANCE1 : MAX_DISTANCE2; const int minMatch = ((int(src[12]) >> 1) & 0x07) + 2; bool res = true; int srcIdx = 13; int dstIdx = 0; int repd0 = count; int repd1 = count; while (true) { const int token = int(src[tkIdx++]); // Get match length and distance int mLen, dist; if ((token & 0x18) == 0) { // Repetition distance, read mLen remainder (if any) outside of token mLen = token & 0x03; if (KANZI_UNLIKELY((mLen == 3) && (mLenIdx >= count))) { res = false; goto exit; } mLen += (mLen == 3 ? minMatch + int(readLength(src, mLenIdx)) : minMatch); dist = (token & 0x04) == 0 ? repd0 : repd1; } else { // Read mLen remainder (if any) outside of token mLen = token & 0x07; if (KANZI_UNLIKELY((mLen == 7) && (mLenIdx >= count))) { res = false; goto exit; } mLen += (mLen == 7 ? minMatch + int(readLength(src, mLenIdx)) : minMatch); if (KANZI_UNLIKELY(mIdx >= count)) { res = false; goto exit; } static const int DIST_SHIFT[4] = { 0, 24, 16, 8 }; static const uint32 DIST_BIAS[4] = { 0, 1, 257, 65793 }; const int width = (token >> 3) & 3; const uint32 value = uint32(BigEndian::readInt32(&src[mIdx])); dist = int((value >> DIST_SHIFT[width]) + DIST_BIAS[width]); mIdx += width; } if (token >= 32) { // Get literal length const uint litLen = (token >= 0xE0) ? uint(7) + readLength(src, srcIdx) : uint(token >> 5); if (KANZI_UNLIKELY((uint(srcIdx) + litLen > uint(litEnd)) || (litLen > uint(dstEnd - dstIdx)))) { res = false; goto exit; } // Emit literals const kanzi::byte* s = &src[srcIdx]; kanzi::byte* d = &dst[dstIdx]; srcIdx += int(litLen); dstIdx += int(litLen); if (KANZI_UNLIKELY(srcIdx >= srcEnd)) { memcpy(d, s, litLen); break; } emitLiterals(s, d, litLen); } repd1 = repd0; repd0 = dist; const int mEnd = dstIdx + mLen; int ref = dstIdx - dist; // Sanity check if (KANZI_UNLIKELY((ref < 0) || (dist > maxDist) || (mEnd > dstEnd))) { res = false; goto exit; } // Copy match if (dist >= 16) { if (mLen >= 64) prefetchRead(&dst[dstIdx + 64]); do { // The stream decoder supplies trailing padding for this // 16-byte copy, which may write up to 15 bytes past mEnd. KANZI_MEM_CP16(&dst[dstIdx], &dst[ref]); ref += 16; dstIdx += 16; } while (dstIdx < mEnd); } else if (dist != 1) { const kanzi::byte* s = &dst[ref]; kanzi::byte* p = &dst[dstIdx]; const kanzi::byte* pend = &p[mLen]; while (p < pend) *p++ = *s++; } else { // dist = 1 memset(&dst[dstIdx], int(dst[ref]), mLen); } dstIdx = mEnd; } exit: output._index += dstIdx; input._index += count; return res && (srcIdx == srcEnd + 13) && (tkIdx == tokenEnd) && (mIdx == matchEnd) && (mLenIdx == count); } template bool LZXCodec::inverseV5(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (count < 13) return false; if (!SliceArray::isValid(input)) throw invalid_argument("LZ codec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("LZ codec: Invalid output block"); const int inputSize = input._length - input._index; // readLength() reads four bytes for a three-byte length encoding. if ((inputSize < READ_LENGTH_GUARD) || (count > inputSize - READ_LENGTH_GUARD)) return false; const int dstEnd = output._length - output._index; kanzi::byte* dst = &output._array[output._index]; const kanzi::byte* src = &input._array[input._index]; int tkIdx = LittleEndian::readInt32(&src[0]); int mIdx = LittleEndian::readInt32(&src[4]); int mLenIdx = LittleEndian::readInt32(&src[8]); // Sanity checks if ((tkIdx < 0) || (mIdx < 0) || (mLenIdx < 0)) return false; if ((tkIdx <= 13) || (tkIdx > count) || (mIdx > count - tkIdx) || (mLenIdx > count - tkIdx - mIdx)) return false; mIdx += tkIdx; mLenIdx += mIdx; const int tokenEnd = mIdx; const int matchEnd = mLenIdx; const int srcEnd = tkIdx - 13; const int litEnd = tkIdx; const int mFlag = int(src[12]) & 1; const int maxDist = (mFlag == 0) ? MAX_DISTANCE1 : MAX_DISTANCE2; const int mmIdx = (int(src[12]) >> 1) & 0x03; const int MIN_MATCHES[4] = { MIN_MATCH4, MIN_MATCH9, MIN_MATCH6, MIN_MATCH6 }; const int minMatch = MIN_MATCHES[mmIdx]; bool res = true; int srcIdx = 13; int dstIdx = 0; int repd0 = 0; int repd1 = 0; while (true) { const int token = int(src[tkIdx++]); if (token >= 32) { // Get literal length const uint litLen = (token >= 0xE0) ? uint(7) + readLength(src, srcIdx) : uint(token >> 5); if ((litLen > uint(dstEnd - dstIdx)) || (litLen > uint(litEnd - srcIdx))) { res = false; goto exit; } // Emit literals const kanzi::byte* s = &src[srcIdx]; kanzi::byte* d = &dst[dstIdx]; srcIdx += int(litLen); dstIdx += int(litLen); if (srcIdx >= srcEnd) { memcpy(d, s, litLen); break; } emitLiterals(s, d, litLen); } // Get match length and distance int mLen = token & 0x0F; int dist; if (mLen == 15) { // Repetition distance, read mLen fully outside of token if (KANZI_UNLIKELY(mLenIdx >= count)) { res = false; goto exit; } mLen = minMatch + int(readLength(src, mLenIdx)); dist = ((token & 0x10) == 0) ? repd0 : repd1; } else { // Read mLen remainder (if any) outside of token if (KANZI_UNLIKELY((mLen == 14) && (mLenIdx >= count))) { res = false; goto exit; } mLen = (mLen == 14) ? 14 + minMatch + int(readLength(src, mLenIdx)) : mLen + minMatch; if (KANZI_UNLIKELY(mIdx >= count)) { res = false; goto exit; } dist = int(src[mIdx++]); if (mFlag != 0) dist = (dist << 8) | int(src[mIdx++]); const int t = (token >> 4) & 1; dist = (dist << (8 * t)) | (-t & int(src[mIdx])); mIdx += t; } prefetchRead(&src[mLenIdx]); repd1 = repd0; repd0 = dist; const int mEnd = dstIdx + mLen; int ref = dstIdx - dist; // Sanity check if ((dist == 0) || (ref < 0) || (dist > maxDist) || (mEnd > dstEnd)) { res = false; goto exit; } prefetchWrite(&dst[dstIdx]); // Copy match if (dist >= 16) { do { // The stream decoder supplies trailing padding for this // 16-byte copy, which may write up to 15 bytes past mEnd. KANZI_MEM_CP16(&dst[dstIdx], &dst[ref]); ref += 16; dstIdx += 16; } while (dstIdx < mEnd); } else if (dist != 1) { const kanzi::byte* s = &dst[ref]; kanzi::byte* p = &dst[dstIdx]; const kanzi::byte* pend = &p[mLen]; while (p < pend) *p++ = *s++; } else { // dist = 1 memset(&dst[dstIdx], int(dst[ref]), mLen); } dstIdx = mEnd; } exit: output._index += dstIdx; input._index += count; return res && (srcIdx == srcEnd + 13) && (tkIdx == tokenEnd) && (mIdx == matchEnd) && (mLenIdx == count); } const uint LZPCodec::HASH_SEED = 0x7FEB352D; const uint LZPCodec::HASH_LOG = 16; const uint LZPCodec::HASH_SHIFT = 32 - HASH_LOG; const int LZPCodec::MIN_MATCH = 64; const int LZPCodec::MIN_BLOCK_LENGTH = 128; const int LZPCodec::MATCH_FLAG = 0xFC; bool LZPCodec::forward(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (count < 4) return false; if (!SliceArray::isValid(input)) throw invalid_argument("LZP codec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("LZP codec: Invalid output block"); if (count > input._length - input._index) return false; if (output._length - output._index < getMaxEncodedLength(count)) return false; // If too small, skip if (count < MIN_BLOCK_LENGTH) return false; const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; const int srcEnd = count; const int dstEnd = count - (count >> 6); if (_hashSize == 0) { const int newSize = 1 << HASH_LOG; int32* hashes = new int32[newSize]; delete[] _hashes; _hashes = hashes; _hashSize = newSize; } memset(_hashes, 0, sizeof(int32) * _hashSize); dst[0] = src[0]; dst[1] = src[1]; dst[2] = src[2]; dst[3] = src[3]; uint ctx = LittleEndian::readInt32(&src[0]); int srcIdx = 4; int dstIdx = 4; while ((srcIdx < srcEnd - MIN_MATCH) && (dstIdx < dstEnd)) { const uint32 h = (HASH_SEED * ctx) >> HASH_SHIFT; const int32 ref = _hashes[h]; _hashes[h] = srcIdx; int bestLen = 0; // Find a match if ((ref != 0) && KANZI_MEM_EQ8(&src[ref + MIN_MATCH - 8], &src[srcIdx + MIN_MATCH - 8])) bestLen = findMatch(src, srcIdx, ref, srcEnd - srcIdx); // No good match ? if (bestLen < MIN_MATCH) { const uint val = uint(src[srcIdx]); ctx = (ctx << 8) | val; dst[dstIdx++] = src[srcIdx++]; if ((ref != 0) && (val == MATCH_FLAG)) { if (dstIdx >= dstEnd) return false; dst[dstIdx++] = kanzi::byte(0xFF); } continue; } srcIdx += bestLen; ctx = LittleEndian::readInt32(&src[srcIdx - 4]); dst[dstIdx++] = kanzi::byte(MATCH_FLAG); bestLen -= MIN_MATCH; // Emit match length while ((bestLen >= 254) && (dstIdx < dstEnd)) { bestLen -= 254; dst[dstIdx++] = kanzi::byte(0xFE); } if (dstIdx >= dstEnd) return false; dst[dstIdx++] = kanzi::byte(bestLen); } while ((srcIdx < srcEnd) && (dstIdx < dstEnd)) { const uint32 h = (HASH_SEED * ctx) >> HASH_SHIFT; const int ref = _hashes[h]; _hashes[h] = srcIdx; const uint val = uint(src[srcIdx]); ctx = (ctx << 8) | val; dst[dstIdx++] = src[srcIdx++]; if ((ref != 0) && (val == MATCH_FLAG)) { if (dstIdx >= dstEnd) return false; dst[dstIdx++] = kanzi::byte(0xFF); } } input._index += srcIdx; output._index += dstIdx; return (srcIdx == count) && (dstIdx < dstEnd); } bool LZPCodec::inverse(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (!SliceArray::isValid(input)) throw invalid_argument("LZP codec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("LZP codec: Invalid output block"); if (count > input._length - input._index) return false; if (count < 4) return false; const int srcEnd = count; const int dstEnd = output._length - output._index; const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; if (dstEnd < count) return false; if (_hashSize == 0) { const int newSize = 1 << HASH_LOG; int32* hashes = new int32[newSize]; delete[] _hashes; _hashes = hashes; _hashSize = newSize; } memset(_hashes, 0, sizeof(int32) * _hashSize); dst[0] = src[0]; dst[1] = src[1]; dst[2] = src[2]; dst[3] = src[3]; uint32 ctx = LittleEndian::readInt32(&dst[0]); int srcIdx = 4; int dstIdx = 4; while (srcIdx < srcEnd) { const int32 h = (HASH_SEED * ctx) >> HASH_SHIFT; int ref = _hashes[h]; _hashes[h] = dstIdx; if ((src[srcIdx] != kanzi::byte(MATCH_FLAG)) || (ref == 0)) { if (dstIdx >= dstEnd) return false; ctx = (ctx << 8) | uint32(src[srcIdx]); dst[dstIdx++] = src[srcIdx++]; continue; } srcIdx++; if (srcIdx >= srcEnd) return false; if (src[srcIdx] == kanzi::byte(0xFF)) { if (dstIdx >= dstEnd) return false; ctx = (ctx << 8) | uint32(MATCH_FLAG); dst[dstIdx++] = kanzi::byte(MATCH_FLAG); srcIdx++; continue; } uint64 mLen = MIN_MATCH; if (src[srcIdx] == kanzi::byte(0xFE)) { while ((srcIdx < srcEnd) && (src[srcIdx] == kanzi::byte(0xFE))) { srcIdx++; mLen += 254; } if (srcIdx >= srcEnd) return false; } mLen += uint64(src[srcIdx++]); if (mLen > uint64(dstEnd - dstIdx)) return false; const int matchLen = int(mLen); const int mEnd = dstIdx + matchLen; if (dstIdx >= ref + 16) { do { // The stream decoder supplies trailing padding for this // 16-byte copy, which may write up to 15 bytes past mEnd. KANZI_MEM_CP16(&dst[dstIdx], &dst[ref]); ref += 16; dstIdx += 16; } while (dstIdx < mEnd); } else { for (int i = 0; i < matchLen; i++) dst[dstIdx + i] = dst[ref + i]; } dstIdx = mEnd; ctx = LittleEndian::readInt32(&dst[dstIdx - 4]); } input._index += srcIdx; output._index += dstIdx; return srcIdx == srcEnd; } kanzi-cpp-2.6.0/src/transform/LZCodec.hpp000066400000000000000000000165361525576310500202350ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_LZCodec #define knz_LZCodec #include "../Context.hpp" #include "../Global.hpp" #include "../Transform.hpp" #include "../Memory.hpp" namespace kanzi { class LZCodec FINAL : public Transform { public: LZCodec(); LZCodec(Context& ctx); ~LZCodec() { delete _delegate; } bool forward(SliceArray& src, SliceArray& dst, int length); bool inverse(SliceArray& src, SliceArray& dst, int length); // Required encoding output buffer size int getMaxEncodedLength(int srcLen) const { return _delegate->getMaxEncodedLength(srcLen); } private: Transform* _delegate; }; // Simple byte oriented LZ77 implementation. template class LZXCodec FINAL : public Transform { public: LZXCodec() { _hashes = nullptr; _hashSize = 0; _tkBuf = nullptr; _mLenBuf = nullptr; _mBuf = nullptr; _bufferSize = 0; _pCtx = nullptr; } LZXCodec(Context& ctx) : _pCtx(&ctx) { _hashes = nullptr; _hashSize = 0; _tkBuf = nullptr; _mLenBuf = nullptr; _mBuf = nullptr; _bufferSize = 0; } ~LZXCodec() { _bufferSize = 0; _hashSize = 0; if (_hashes != nullptr) delete[] _hashes; if (_mLenBuf != nullptr) delete[] _mLenBuf; if (_mBuf != nullptr) delete[] _mBuf; if (_tkBuf != nullptr) delete[] _tkBuf; } bool forward(SliceArray& src, SliceArray& dst, int length); bool inverse(SliceArray& src, SliceArray& dst, int length); // Required encoding output buffer size int getMaxEncodedLength(int srcLen) const { // readLength() may read two bytes past the encoded block return ((srcLen <= 1024) ? srcLen + 16 : srcLen + (srcLen / 64)) + 4; } private: static const uint HASH_SEED; static const uint HASH_LOG; static const uint HASH_LSHIFT; static const uint HASH_RSHIFT; static const int MAX_DISTANCE1; static const int MAX_DISTANCE2; static const int MIN_MATCH4; static const int MIN_MATCH6; static const int MIN_MATCH9; static const int MAX_MATCH; static const int MIN_BLOCK_LENGTH; static const int READ_LENGTH_GUARD; int32* _hashes; int _hashSize; byte* _mLenBuf; byte* _mBuf; byte* _tkBuf; int _bufferSize; Context* _pCtx; bool inverseV7(SliceArray& src, SliceArray& dst, int length); bool inverseV6(SliceArray& src, SliceArray& dst, int length); bool inverseV5(SliceArray& src, SliceArray& dst, int length); static int emitLength(byte block[], int len); static void emitLiterals(const byte src[], byte dst[], int len); static int findMatch(const byte block[], const int pos, const int ref, const int maxMatch); static uint readLength(const byte block[], int& pos); static int32 hash(const byte* p); }; class LZPCodec FINAL : public Transform { public: LZPCodec() { _hashes = nullptr; _hashSize = 0; } LZPCodec(Context&) { _hashes = nullptr; _hashSize = 0; } ~LZPCodec() { delete[] _hashes; } bool forward(SliceArray& src, SliceArray& dst, int length); bool inverse(SliceArray& src, SliceArray& dst, int length); // Required encoding output buffer size int getMaxEncodedLength(int srcLen) const { return (srcLen <= 1024) ? srcLen + 16 : srcLen + (srcLen / 64); } private: static const uint HASH_SEED; static const uint HASH_LOG; static const uint HASH_SHIFT; static const int MIN_MATCH; static const int MIN_BLOCK_LENGTH; static const int MATCH_FLAG; int32* _hashes; int _hashSize; static int findMatch(const byte block[], const int pos, const int ref, const int maxMatch); }; template inline void LZXCodec::emitLiterals(const byte src[], byte dst[], int len) { // Callers provide trailing padding for this 16-byte copy, which may // read and write up to 15 bytes past the requested literal length. for (int i = 0; i < len; i += 16) KANZI_MEM_CP16(&dst[i], &src[i]); } template inline int32 LZXCodec::hash(const byte* p) { return ((uint64(LittleEndian::readLong64(p)) << HASH_LSHIFT) * HASH_SEED) >> HASH_RSHIFT; } template inline int LZXCodec::emitLength(byte block[], int length) { if (length < 254) { block[0] = byte(length); return 1; } if (length < 65536 + 254) { const uint32 l = (length - 254) | 0x00FE0000; kanzi::BigEndian::writeInt32(&block[0], l << 8); return 3; } const uint32 l = (length - 255) | 0xFF000000; kanzi::BigEndian::writeInt32(&block[0], l); return 4; } template inline uint LZXCodec::readLength(const byte block[], int& pos) { uint res = uint(block[pos++]); if (res < 254) return res; if (res == 254) { res += uint((kanzi::BigEndian::readInt16(&block[pos])) & 0xFFFF); pos += 2; return res; } res += (uint32(kanzi::BigEndian::readInt32(&block[pos])) >> 8); pos += 3; return res; } template inline int LZXCodec::findMatch(const byte src[], const int srcIdx, const int ref, const int maxMatch) { int n = 0; while (n + 8 <= maxMatch) { const int64 diff = LittleEndian::readLong64(&src[srcIdx + n]) ^ LittleEndian::readLong64(&src[ref + n]); if (diff != 0) { n += (Global::trailingZeros(uint64(diff)) >> 3); break; } n += 8; } return n; } inline int LZPCodec::findMatch(const byte src[], const int srcIdx, const int ref, const int maxMatch) { int n = 0; while (n + 8 <= maxMatch) { const int64 diff = LittleEndian::readLong64(&src[srcIdx + n]) ^ LittleEndian::readLong64(&src[ref + n]); if (diff != 0) { n += (Global::trailingZeros(uint64(diff)) >> 3); break; } n += 8; } return n; } } #endif kanzi-cpp-2.6.0/src/transform/NullTransform.hpp000066400000000000000000000040741525576310500215520ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_NullTransform #define knz_NullTransform #include #include "../Context.hpp" #include "../Transform.hpp" namespace kanzi { class NullTransform FINAL : public Transform { public: NullTransform() {} NullTransform(Context&) {} ~NullTransform() {} bool forward(SliceArray& input, SliceArray& output, int length) { return doCopy(input, output, length); } bool inverse(SliceArray& input, SliceArray& output, int length) { return doCopy(input, output, length); } // Required encoding output buffer size int getMaxEncodedLength(int inputLen) const { return inputLen; } private: bool doCopy(SliceArray& input, SliceArray& output, int length) const; }; inline bool NullTransform::doCopy(SliceArray& input, SliceArray& output, int length) const { if (length == 0) return true; if (!SliceArray::isValid(input)) throw std::invalid_argument("Invalid input block"); if (!SliceArray::isValid(output)) throw std::invalid_argument("Invalid output block"); if (input._index + length > input._length) return false; if (output._index + length > output._length) return false; std::memmove(&output._array[output._index], &input._array[input._index], size_t(length)); input._index += length; output._index += length; return true; } } #endif kanzi-cpp-2.6.0/src/transform/RLT.cpp000066400000000000000000000240061525576310500173750ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include "RLT.hpp" #include "../Global.hpp" #include "../Memory.hpp" #include "../util/strings.hpp" using namespace kanzi; using namespace std; const int RLT::RUN_LEN_ENCODE1 = 224; // used to encode run length const int RLT::RUN_LEN_ENCODE2 = (255 - RUN_LEN_ENCODE1) << 8; // used to encode run length const int RLT::RUN_THRESHOLD = 3; const int RLT::MAX_RUN = 0xFFFF + RUN_LEN_ENCODE2 + RUN_THRESHOLD - 1; const int RLT::MAX_RUN4 = MAX_RUN - 4; const int RLT::MIN_BLOCK_LENGTH = 16; const kanzi::byte RLT::DEFAULT_ESCAPE = kanzi::byte(0xFB); bool RLT::forward(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (count < MIN_BLOCK_LENGTH) return false; if (!SliceArray::isValid(input)) throw invalid_argument("RLT: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("RLT: Invalid output block"); if (output._length - output._index < getMaxEncodedLength(count)) return false; const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; Global::DataType dt = Global::UNDEFINED; bool findBestEscape = true; if (_pCtx != nullptr) { dt = (Global::DataType) _pCtx->getInt("dataType", Global::UNDEFINED); if ((dt == Global::DNA) || (dt == Global::BASE64) || (dt == Global::UTF8)) return false; std::string entropyType = _pCtx->getString("entropy"); transform(entropyType.begin(), entropyType.end(), entropyType.begin(), safeToUpper); // Fast track if fast entropy coder is used if ((entropyType == "NONE") || (entropyType == "ANS0") || (entropyType == "HUFFMAN") || (entropyType == "RANGE")) findBestEscape = false; } kanzi::byte escape = DEFAULT_ESCAPE; if (findBestEscape == true) { uint freqs[256] = { 0 }; Global::computeHistogram(&src[0], count, freqs); if (dt == Global::UNDEFINED) { dt = Global::detectSimpleType(count, freqs); if ((_pCtx != nullptr) && (dt != Global::UNDEFINED)) _pCtx->putInt("dataType", dt); if ((dt == Global::DNA) || (dt == Global::BASE64) || (dt == Global::UTF8)) return false; } int minIdx = 0; if (freqs[minIdx] > 0) { for (int i = 1; i < 256; i++) { if (freqs[i] < freqs[minIdx]) { minIdx = i; if (freqs[i] == 0) break; } } } escape = kanzi::byte(minIdx); } int srcIdx = 0; int dstIdx = 0; const int srcEnd = count; const int srcEnd4 = srcEnd - 4; const int dstEnd = output._length; bool res = true; int run = 0; kanzi::byte prev = src[srcIdx++]; dst[dstIdx++] = escape; dst[dstIdx++] = prev; if (prev == escape) dst[dstIdx++] = kanzi::byte(0); // Main loop while (true) { if (prev == src[srcIdx]) { const uint32 v = 0x01010101u * uint32(prev); const uint32 diff = uint32(LittleEndian::readInt32(&src[srcIdx])) ^ v; if (diff == 0) { srcIdx += 4; run += 4; if ((run < MAX_RUN4) && (srcIdx < srcEnd4)) continue; } else { const int n = Global::trailingZeros(diff) >> 3; srcIdx += n; run += n; } } if (run > RUN_THRESHOLD) { if (dstIdx + 6 >= dstEnd) { res = false; break; } dstIdx += emitRunLength(&dst[dstIdx], run, escape, prev); } else if (prev != escape) { if (dstIdx + run >= dstEnd) { res = false; break; } if (run-- > 0) dst[dstIdx++] = prev; while (run-- > 0) dst[dstIdx++] = prev; } else { // escape literal if (dstIdx + (2 * run) >= dstEnd) { res = false; break; } while (run-- > 0) { dst[dstIdx++] = escape; dst[dstIdx++] = kanzi::byte(0); } } prev = src[srcIdx]; srcIdx++; run = 1; if (srcIdx >= srcEnd4) break; } if (res == true) { // run == 1 if (prev != escape) { if (dstIdx + run < dstEnd) { while (run-- > 0) dst[dstIdx++] = prev; } } else { // escape literal if (dstIdx + (2 * run) < dstEnd) { while (run-- > 0) { dst[dstIdx++] = escape; dst[dstIdx++] = kanzi::byte(0); } } } // Emit the last few bytes while ((srcIdx < srcEnd) && (dstIdx < dstEnd)) { if (src[srcIdx] == escape) { if (dstIdx + 2 >= dstEnd) { res = false; break; } dst[dstIdx++] = escape; dst[dstIdx++] = kanzi::byte(0); srcIdx++; continue; } dst[dstIdx++] = src[srcIdx++]; } res &= (srcIdx == srcEnd); } input._index += srcIdx; output._index += dstIdx; return res && (dstIdx < srcIdx); } int RLT::emitRunLength(kanzi::byte dst[], int run, kanzi::byte escape, kanzi::byte val) { dst[0] = val; dst[1] = kanzi::byte(0); int dstIdx = (val == escape) ? 2 : 1; dst[dstIdx++] = escape; run -= RUN_THRESHOLD; // Encode run length if (run >= RUN_LEN_ENCODE1) { if (run < RUN_LEN_ENCODE2) { run -= RUN_LEN_ENCODE1; dst[dstIdx++] = kanzi::byte(RUN_LEN_ENCODE1 + (run >> 8)); } else { run -= RUN_LEN_ENCODE2; dst[dstIdx++] = kanzi::byte(0xFF); dst[dstIdx++] = kanzi::byte(run >> 8); } } dst[dstIdx] = kanzi::byte(run); return dstIdx + 1; } bool RLT::inverse(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (!SliceArray::isValid(input)) throw invalid_argument("RLT: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("RLT: Invalid output block"); if (input._index + count > input._length) return false; const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; int srcIdx = 0; int dstIdx = 0; const int srcEnd = srcIdx + count; const int dstEnd = output._length - output._index; bool res = true; const kanzi::byte escape = src[srcIdx++]; if ((srcIdx < srcEnd) && (src[srcIdx] == escape)) { srcIdx++; // The data cannot start with a run but may start with an escape literal if ((srcIdx < srcEnd) && (src[srcIdx] != kanzi::byte(0))) return false; if (dstIdx >= dstEnd) return false; dst[dstIdx++] = escape; srcIdx++; } // Main loop while (srcIdx < srcEnd) { // Copy a span of literals in one operation. Escaped literals are // encoded as escape, 0, so the first escape always terminates the // current literal span. const byte* const esc = static_cast( std::memchr(&src[srcIdx], int(uint8(escape)), size_t(srcEnd - srcIdx))); const int literalLen = (esc == nullptr) ? (srcEnd - srcIdx) : int(esc - &src[srcIdx]); if (literalLen > 0) { if (literalLen > dstEnd - dstIdx) { res = false; break; } std::memcpy(&dst[dstIdx], &src[srcIdx], size_t(literalLen)); srcIdx += literalLen; dstIdx += literalLen; } if (srcIdx >= srcEnd) break; // An escape marker was found srcIdx++; if (srcIdx >= srcEnd) { res = false; break; } int run = int(src[srcIdx++]); if (run == 0) { // Just an escape symbol, not a run if (dstIdx >= dstEnd) { res = false; break; } dst[dstIdx++] = escape; continue; } // Decode run length if (run == 0xFF) { if (srcIdx + 1 >= srcEnd) { res = false; break; } run = (int(src[srcIdx]) << 8) | int(src[srcIdx + 1]); srcIdx += 2; run += RUN_LEN_ENCODE2; } else if (run >= RUN_LEN_ENCODE1) { if (srcIdx >= srcEnd) { res = false; break; } run = ((run - RUN_LEN_ENCODE1) << 8) | int(src[srcIdx]); srcIdx++; run += RUN_LEN_ENCODE1; } run += (RUN_THRESHOLD - 1); if ((dstIdx + run > dstEnd) || (run > MAX_RUN)) { res = false; break; } if (dstIdx == 0) { res = false; break; } memset(&dst[dstIdx], int(dst[dstIdx - 1]), size_t(run)); dstIdx += run; } input._index += srcIdx; output._index += dstIdx; return res && (srcIdx == srcEnd); } kanzi-cpp-2.6.0/src/transform/RLT.hpp000066400000000000000000000033711525576310500174040ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_RLT #define knz_RLT #include "../Context.hpp" #include "../Transform.hpp" namespace kanzi { // Implementation of an escaped RLE // Run length encoding: // RUN_LEN_ENCODE1 = 224 => RUN_LEN_ENCODE2 = 31*224 = 6944 // 4 <= runLen < 224+4 -> 1 byte // 228 <= runLen < 6944+228 -> 2 bytes // 7172 <= runLen < 65535+7172 -> 3 bytes class RLT FINAL : public Transform { public: RLT() { _pCtx = nullptr; } RLT(Context& ctx) : _pCtx(&ctx) {} ~RLT() {} bool forward(SliceArray& pSrc, SliceArray& pDst, int length); bool inverse(SliceArray& pSrc, SliceArray& pDst, int length); int getMaxEncodedLength(int srcLen) const { return (srcLen <= 512) ? srcLen + 32 : srcLen; } private: static const int RUN_LEN_ENCODE1; static const int RUN_LEN_ENCODE2; static const int RUN_THRESHOLD; static const int MAX_RUN; static const int MAX_RUN4; static const int MIN_BLOCK_LENGTH; static const byte DEFAULT_ESCAPE; static int emitRunLength(byte dst[], int run, byte escape, byte val); Context* _pCtx; }; } #endif kanzi-cpp-2.6.0/src/transform/ROLZCodec.cpp000066400000000000000000001116401525576310500204610ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include "ROLZCodec.hpp" #include "../Global.hpp" #include "../Memory.hpp" #include "../bitstream/DefaultInputBitStream.hpp" #include "../bitstream/DefaultOutputBitStream.hpp" #include "../entropy/ANSRangeDecoder.hpp" #include "../entropy/ANSRangeEncoder.hpp" #include "../util/fixedbuf.hpp" using namespace kanzi; using namespace std; const int ROLZCodec::HASH_SIZE = 65536; const int ROLZCodec::CHUNK_SIZE = 16 * 1024 * 1024; const int32 ROLZCodec::HASH = 200002979; const int32 ROLZCodec::HASH_MASK = ~(ROLZCodec::CHUNK_SIZE - 1); const int ROLZCodec::MAX_BLOCK_SIZE = 1024 * 1024 * 1024; const int ROLZCodec::MIN_BLOCK_SIZE = 64; ROLZCodec::ROLZCodec(uint logPosChecks) { _delegate = new ROLZCodec1(logPosChecks); } ROLZCodec::ROLZCodec(Context& ctx) { string transform = ctx.getString("transform", "NONE"); _delegate = (transform.find("ROLZX") != string::npos) ? static_cast*>(new ROLZCodec2(ctx)) : static_cast*>(new ROLZCodec1(ctx)); } bool ROLZCodec::forward(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (count < MIN_BLOCK_SIZE) return false; if (!SliceArray::isValid(input)) throw invalid_argument("ROLZ codec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("ROLZ codec: Invalid output block"); if (input._array == output._array) return false; if (count > MAX_BLOCK_SIZE) return false; return _delegate->forward(input, output, count); } bool ROLZCodec::inverse(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (!SliceArray::isValid(input)) throw invalid_argument("ROLZ codec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("ROLZ codec: Invalid output block"); if (input._array == output._array) return false; if ((count < 5) || (input._index + count > input._length)) return false; if (count > MAX_BLOCK_SIZE) return false; return _delegate->inverse(input, output, count); } const int ROLZCodec1::MIN_MATCH3 = 3; const int ROLZCodec1::MIN_MATCH4 = 4; const int ROLZCodec1::MIN_MATCH7 = 7; const int ROLZCodec1::MAX_MATCH = MIN_MATCH3 + 65535; const int ROLZCodec1::LOG_POS_CHECKS = 4; ROLZCodec1::ROLZCodec1(uint logPosChecks) : _logPosChecks(logPosChecks) { if ((logPosChecks < 2) || (logPosChecks > 8)) { stringstream ss; ss << "ROLZ codec: Invalid logPosChecks parameter: " << logPosChecks << " (must be in [2..8])"; throw invalid_argument(ss.str()); } _pCtx = nullptr; _posChecks = 1 << _logPosChecks; _maskChecks = uint8(_posChecks - 1); _minMatch = MIN_MATCH3; _mSize = 0; _matches = nullptr; memset(&_counters[0], 0, sizeof(_counters)); } ROLZCodec1::ROLZCodec1(Context& ctx) : _pCtx(&ctx) { _logPosChecks = LOG_POS_CHECKS; _posChecks = 1 << _logPosChecks; _maskChecks = uint8(_posChecks - 1); _minMatch = MIN_MATCH3; _mSize = 0; _matches = nullptr; memset(&_counters[0], 0, sizeof(_counters)); } // return position index (_logPosChecks bits) + length (16 bits) or -1 int ROLZCodec1::findMatch(const kanzi::byte buf[], int pos, int end, uint32 hash32, const uint32* matches, const uint8* counter) const { const int s = int(*counter); const int e = s - _posChecks; prefetchRead(matches); const kanzi::byte* curBuf = &buf[pos]; int bestLen = 0; int bestIdx = -1; const int maxMatch = min(ROLZCodec1::MAX_MATCH, end - pos) - 8; // Check all recorded positions for (int i = s; i > e; i--) { uint32 ref = matches[i & _maskChecks]; // Hash check may save a memory access ... if ((ref & ROLZCodec::HASH_MASK) != hash32) continue; ref &= ~ROLZCodec::HASH_MASK; if (buf[ref + bestLen] != curBuf[bestLen]) continue; int n = 0; while (n < maxMatch) { const int64 diff = LittleEndian::readLong64(&buf[ref + n]) ^ LittleEndian::readLong64(&curBuf[n]); if (diff != 0) { n += (Global::trailingZeros(uint64(diff)) >> 3); break; } n += 8; } if (n > bestLen) { bestIdx = i; bestLen = n; } } return (bestLen < _minMatch) ? -1 : ((s - bestIdx) << 16) | (bestLen - _minMatch); } bool ROLZCodec1::forward(SliceArray& input, SliceArray& output, int count) { const int outputIndex = output._index; const int outputCapacity = output._length - outputIndex; if (outputCapacity < getMaxEncodedLength(count)) return false; const int srcEnd = count - 4; const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[outputIndex]; BigEndian::writeInt32(&dst[0], count); int dstIdx = 5; int sizeChunk = min(count, ROLZCodec::CHUNK_SIZE); int startChunk = 0; const int litBufSize = getMaxEncodedLength(sizeChunk); const int lenBufSize = sizeChunk / 5; const int mIdxBufSize = sizeChunk / 4; const int tkBufSize = sizeChunk / 4; kanzi::byte* arena = new kanzi::byte[size_t(litBufSize) + lenBufSize + mIdxBufSize + tkBufSize]; SliceArray litBuf(&arena[0], litBufSize); SliceArray lenBuf(&arena[litBufSize], lenBufSize); SliceArray mIdxBuf(&arena[litBufSize + lenBufSize], mIdxBufSize); SliceArray tkBuf(&arena[litBufSize + lenBufSize + mIdxBufSize], tkBufSize); memset(&_counters[0], 0, sizeof(_counters)); bool success = true; const int litOrder = (count < (1 << 17)) ? 0 : 1; int flags = litOrder; _minMatch = MIN_MATCH3; int delta = 2; if (_pCtx != nullptr) { Global::DataType dt = (Global::DataType) _pCtx->getInt("dataType", Global::UNDEFINED); if (dt == Global::UNDEFINED) { uint freqs0[256] = { 0 }; Global::computeHistogram(&src[0], count, freqs0); dt = Global::detectSimpleType(count, freqs0); if (dt != Global::UNDEFINED) _pCtx->putInt("dataType", dt); } if (dt == Global::EXE) { delta = 3; flags |= 8; } else if (dt == Global::DNA) { delta = 8; _minMatch = MIN_MATCH7; flags |= 4; } else if (dt == Global::MULTIMEDIA) { delta = 8; _minMatch = MIN_MATCH4; flags |= 2; } } if (_mSize == 0) { const size_t newSize = size_t(ROLZCodec::HASH_SIZE << _logPosChecks); uint32* matches = new uint32[newSize]; delete[] _matches; _matches = matches; _mSize = newSize; } flags |= (_logPosChecks << 4); dst[4] = kanzi::byte(flags); const bool cond = _minMatch == MIN_MATCH3; // Main loop while (startChunk < srcEnd) { litBuf._index = 0; lenBuf._index = 0; mIdxBuf._index = 0; tkBuf._index = 0; memset(&_matches[0], 0, sizeof(uint32) * size_t(ROLZCodec::HASH_SIZE << _logPosChecks)); const int endChunk = min(startChunk + sizeChunk, srcEnd); sizeChunk = endChunk - startChunk; const kanzi::byte* buf = &src[startChunk]; const kanzi::byte* ref = &src[startChunk - delta]; int srcIdx = 0; const int n = min(srcEnd - startChunk, 8); for (int j = 0; j < n; j++) litBuf._array[litBuf._index++] = buf[srcIdx++]; int firstLitIdx = srcIdx; int srcInc = 0; while (srcIdx < sizeChunk) { const uint32 key = (cond == true) ? ROLZCodec::getKey1(&ref[srcIdx]): ROLZCodec::getKey2(&ref[srcIdx]); uint8* counter = &_counters[key]; uint32* matches = &_matches[key << _logPosChecks]; uint32 hash32 = ROLZCodec::hash(&buf[srcIdx]); int match = findMatch(buf, srcIdx, sizeChunk, hash32, matches, counter); // Register current position *counter = (*counter + 1) & _maskChecks; matches[*counter] = hash32 | int32(srcIdx); if (match < 0) { srcIdx++; srcIdx += (srcInc >> 6); srcInc++; continue; } // Check if better match at next position const int srcIdx1 = srcIdx + 1; const uint32 key2 = (cond == true) ? ROLZCodec::getKey1(&ref[srcIdx1]) : ROLZCodec::getKey2(&ref[srcIdx1]); counter = &_counters[key2]; matches = &_matches[key2 << _logPosChecks]; hash32 = ROLZCodec::hash(&buf[srcIdx1]); const int match2 = findMatch(buf, srcIdx1, sizeChunk, hash32, matches, counter); if ((match2 >= 0) && ((match2 & 0xFFFF) > (match & 0xFFFF))) { // New match is better match = match2; srcIdx = srcIdx1; // Register current position *counter = (*counter + 1) & _maskChecks; matches[*counter] = hash32 | int32(srcIdx); } // token LLLLLMMM -> L lit length, M match length const int litLen = srcIdx - firstLitIdx; const int token = (litLen < 31) ? litLen << 3 : 0xF8; const int mLen = match & 0xFFFF; if (mLen >= 7) { tkBuf._array[tkBuf._index++] = kanzi::byte(token | 0x07); lenBuf._index += emitLength(&lenBuf._array[lenBuf._index], mLen - 7); } else { tkBuf._array[tkBuf._index++] = kanzi::byte(token | mLen); } // Emit literals if (litLen > 0) { lenBuf._index += (litLen >= 31 ? emitLength(&lenBuf._array[lenBuf._index], litLen - 31) : 0); memcpy(&litBuf._array[litBuf._index], &buf[firstLitIdx], size_t(litLen)); litBuf._index += litLen; } // Emit match index mIdxBuf._array[mIdxBuf._index++] = kanzi::byte(match >> 16); srcIdx += (mLen + _minMatch); firstLitIdx = srcIdx; srcInc = 0; } // Emit last chunk literals const int litLen = sizeChunk - firstLitIdx; if (tkBuf._index != 0) { // At least one match to emit const int token = (litLen < 31) ? litLen << 3 : 0xF8; tkBuf._array[tkBuf._index++] = kanzi::byte(token); } lenBuf._index += (litLen >= 31 ? emitLength(&lenBuf._array[lenBuf._index], litLen - 31) : 0); memcpy(&litBuf._array[litBuf._index], &buf[firstLitIdx], size_t(litLen)); litBuf._index += litLen; try { // Encode literal, match length and match index buffers const int remaining = outputCapacity - dstIdx; if (remaining <= 0) { input._index = startChunk + srcIdx; success = false; goto End; } ofixedbuf buffer(reinterpret_cast(&dst[dstIdx]), size_t(remaining)); ostream os(&buffer); try { // The bitstream must be scoped so that its destructor flushes // the final partial word before buffer.written() is queried. DefaultOutputBitStream obs(os, 65536); obs.writeBits(litBuf._index, 32); obs.writeBits(tkBuf._index, 32); obs.writeBits(lenBuf._index, 32); obs.writeBits(mIdxBuf._index, 32); ANSRangeEncoder litEnc(obs, litOrder); litEnc.encode(litBuf._array, 0, litBuf._index); litEnc.dispose(); ANSRangeEncoder mEnc(obs, 0, 32768); mEnc.encode(tkBuf._array, 0, tkBuf._index); mEnc.encode(lenBuf._array, 0, lenBuf._index); mEnc.encode(mIdxBuf._array, 0, mIdxBuf._index); mEnc.dispose(); } catch (const BitStreamException&) { // A bounded output stream reports a full destination as an // I/O failure. Preserve the transform's usual false result // for that case, while propagating other bitstream errors. if (os.fail() == true) { input._index = startChunk + srcIdx; success = false; goto End; } throw; } if (os.fail() == true) { input._index = startChunk + srcIdx; success = false; goto End; } dstIdx += int(buffer.written()); } catch (...) { delete[] arena; throw; } startChunk = endChunk; } End: if (success == true) { if (dstIdx + 4 > outputCapacity) { input._index = srcEnd; } else { // Emit last literals memcpy(&dst[dstIdx], &src[srcEnd], 4); dstIdx += 4; input._index = srcEnd + 4; } } output._index += dstIdx; delete[] arena; return (input._index == count) && (dstIdx < count); } bool ROLZCodec1::inverse(SliceArray& input, SliceArray& output, int count) { kanzi::byte* src = &input._array[input._index]; const int end = BigEndian::readInt32(&src[0]); if ((end <= 4) || (end - 4 > output._length - output._index)) return false; const int dstEnd = end - 4; int srcIdx = 5; int sizeChunk = min(dstEnd, ROLZCodec::CHUNK_SIZE); int startChunk = 0; const int flags = int(src[4]); const int litOrder = flags & 1; _minMatch = MIN_MATCH3; int delta = 2; switch (flags & 0x0E) { case 2: // MULTIMEDIA _minMatch = MIN_MATCH4; delta = 8; break; case 4: // DNA _minMatch = MIN_MATCH7; delta = 8; break; case 8: // EXE delta = 3; break; default: break; } _logPosChecks = flags >> 4; if ((_logPosChecks < 2) || (_logPosChecks > 8)) return false; if (_mSize < size_t(ROLZCodec::HASH_SIZE << _logPosChecks)) { const size_t newSize = size_t(ROLZCodec::HASH_SIZE << _logPosChecks); uint32* matches = new uint32[newSize]; delete[] _matches; _matches = matches; _mSize = newSize; } _posChecks = 1 << _logPosChecks; _maskChecks = uint8(_posChecks - 1); const int litBufSize = sizeChunk; const int mIdxBufSize = sizeChunk / 4; const int tkBufSize = sizeChunk / 4; const int lenBufSize = sizeChunk / 5; // Pad the length buffer so readLength() can safely read up to 4 bytes // after the logical end once the first byte has been validated. kanzi::byte* arena = new kanzi::byte[litBufSize + mIdxBufSize + tkBufSize + lenBufSize + 4]; SliceArray litBuf(&arena[0], litBufSize); SliceArray mIdxBuf(&arena[litBufSize], mIdxBufSize); SliceArray tkBuf(&arena[litBufSize + mIdxBufSize], tkBufSize); SliceArray lenBuf(&arena[litBufSize + mIdxBufSize + tkBufSize], lenBufSize); memset(&lenBuf._array[lenBufSize], 0, 4); memset(&_counters[0], 0, sizeof(_counters)); bool success = true; // Main loop while (startChunk < dstEnd) { litBuf._index = 0; lenBuf._index = 0; mIdxBuf._index = 0; tkBuf._index = 0; const int endChunk = min(startChunk + sizeChunk, dstEnd); sizeChunk = endChunk - startChunk; bool onlyLiterals = false; int litLenDecoded = 0; int tkLen = 0; int mLenLen = 0; int mIdxLen = 0; try { // Decode literal, length and match index buffers if (srcIdx >= count) { success = false; goto End; } ifixedbuf buffer(reinterpret_cast(&src[srcIdx]), size_t(count - srcIdx)); istream is(&buffer); DefaultInputBitStream ibs(is, 65536); const int litLen = int(ibs.readBits(32)); tkLen = int(ibs.readBits(32)); mLenLen = int(ibs.readBits(32)); mIdxLen = int(ibs.readBits(32)); const int firstLitLen = min(sizeChunk, 8); if ((litLen < 0) || (tkLen < 0) || (mLenLen < 0) || (mIdxLen < 0)) { input._index += srcIdx; output._index += startChunk; success = false; goto End; } if ((litLen > litBuf._length) || (tkLen > tkBuf._length) || (mLenLen > lenBuf._length) || (mIdxLen > mIdxBuf._length) || (litLen < firstLitLen) || (litLen > sizeChunk) || ((tkLen == 0) && (mIdxLen != 0)) || ((tkLen > 0) && (mIdxLen + 1 != tkLen))) { input._index += srcIdx; output._index += startChunk; success = false; goto End; } litLenDecoded = litLen; ANSRangeDecoder litDec(ibs, litOrder); litDec.decode(litBuf._array, 0, litLen); litDec.dispose(); ANSRangeDecoder mDec(ibs, 0, 32768); mDec.decode(tkBuf._array, 0, tkLen); mDec.decode(lenBuf._array, 0, mLenLen); mDec.decode(mIdxBuf._array, 0, mIdxLen); mDec.dispose(); onlyLiterals = tkLen == 0; srcIdx += int((ibs.read() + 7) >> 3); } catch (const BitStreamException&) { delete[] arena; throw; } if (onlyLiterals == true) { // Shortcut when no match if (litLenDecoded != sizeChunk) { success = false; goto End; } memcpy(&output._array[output._index], &litBuf._array[0], size_t(sizeChunk)); startChunk = endChunk; output._index += sizeChunk; continue; } memset(&_matches[0], 0, sizeof(uint32) * size_t(ROLZCodec::HASH_SIZE << _logPosChecks)); const bool cond = _minMatch == MIN_MATCH3; kanzi::byte* buf = &output._array[output._index]; const kanzi::byte* refBuf = &output._array[output._index - delta]; int dstIdx = 0; const int n = min(dstEnd - output._index, 8); for (int j = 0; j < n; j++) buf[dstIdx++] = litBuf._array[litBuf._index++]; // Next chunk while (dstIdx < sizeChunk) { // token LLLLLMMM -> L lit length, M match length const int token = int(tkBuf._array[tkBuf._index++]); int mLen = token & 0x07; if (mLen == 7) { if (KANZI_UNLIKELY(lenBuf._index >= mLenLen)) { success = false; goto End; } mLen += _minMatch + readLength(lenBuf._array, lenBuf._index); } else { mLen += _minMatch; } // Emit literals int litLen = token >> 3; if (token >= 0xF8) { if (KANZI_UNLIKELY(lenBuf._index >= mLenLen)) { success = false; goto End; } litLen = readLength(lenBuf._array, lenBuf._index) + 31; } if (litLen > 0) { if (KANZI_UNLIKELY(dstIdx + litLen > litBufSize)) { success = false; goto End; } memcpy(&buf[dstIdx], &litBuf._array[litBuf._index], size_t(litLen)); int srcInc = 0; if (cond == true) { for (int k = 0; k < litLen; k++) { const uint32 key = ROLZCodec::getKey1(&refBuf[dstIdx + k]); uint8* counter = &_counters[key]; uint32* matches = &_matches[key << _logPosChecks]; *counter = (*counter + 1) & _maskChecks; matches[*counter] = dstIdx + k; k += (srcInc >> 6); srcInc++; } } else { for (int k = 0; k < litLen; k++) { const uint32 key = ROLZCodec::getKey2(&refBuf[dstIdx + k]); uint8* counter = &_counters[key]; uint32* matches = &_matches[key << _logPosChecks]; *counter = (*counter + 1) & _maskChecks; matches[*counter] = dstIdx + k; k += (srcInc >> 6); srcInc++; } } litBuf._index += litLen; dstIdx += litLen; if (litLen >= 64) prefetchRead(&litBuf._array[litBuf._index]); if (KANZI_UNLIKELY(dstIdx >= sizeChunk)) { // Last chunk literals not followed by match if (dstIdx == sizeChunk) break; output._index += dstIdx; success = false; goto End; } } // Sanity check if (KANZI_UNLIKELY(output._index + dstIdx + mLen > dstEnd)) { success = false; goto End; } const uint8 mIdx = uint8(mIdxBuf._array[mIdxBuf._index++]); const uint32 key = (cond == true) ? ROLZCodec::getKey1(&refBuf[dstIdx]) : ROLZCodec::getKey2(&refBuf[dstIdx]); uint32* matches = &_matches[key << _logPosChecks]; const int32 ref = matches[(_counters[key] - mIdx) & _maskChecks]; _counters[key] = (_counters[key] + 1) & _maskChecks; matches[_counters[key]] = dstIdx; if ((dstIdx - ref >= 64) && (mLen >= 64)) prefetchRead(&buf[ref + 64]); dstIdx = ROLZCodec::emitCopy(buf, dstIdx, ref, mLen); } // Extra validation that all buffer elements have been consumsed if ((tkBuf._index != tkLen) || (mIdxBuf._index != mIdxLen) || (litBuf._index != litLenDecoded) || (lenBuf._index != mLenLen)) { success = false; goto End; } startChunk = endChunk; output._index += dstIdx; } End: if (success == true) { // A valid ROLZ block must leave exactly 4 raw tail bytes. if ((output._index + 4 > output._length) || (count - srcIdx != 4)) { success = false; } else { memcpy(&output._array[output._index], &src[srcIdx], 4); output._index += 4; srcIdx += 4; } } input._index += srcIdx; delete[] arena; return (success == true) && (srcIdx == count); } const uint64 ROLZEncoder::TOP = 0x00FFFFFFFFFFFFFF; const uint64 ROLZEncoder::MASK_0_32 = 0x00000000FFFFFFFF; const int ROLZEncoder::MATCH_FLAG = 0; const int ROLZEncoder::LITERAL_FLAG = 1; const int ROLZEncoder::PSCALE = 0xFFFF; ROLZEncoder::ROLZEncoder(uint litLogSize, uint mLogSize, kanzi::byte buf[], int& idx) : _idx(idx) , _low(0) , _high(TOP) , _c1(1) , _ctx(0) , _pIdx(LITERAL_FLAG) { _buf = buf; _logSizes[MATCH_FLAG] = mLogSize; _logSizes[LITERAL_FLAG] = litLogSize; _probs[MATCH_FLAG] = new uint16[256 << mLogSize]; _probs[LITERAL_FLAG] = new uint16[256 << litLogSize]; reset(); } void ROLZEncoder::reset() { const int mLogSize = _logSizes[MATCH_FLAG]; for (int i = 0; i < (256 << mLogSize); i++) _probs[MATCH_FLAG][i] = PSCALE >> 1; const int litLogSize = _logSizes[LITERAL_FLAG]; for (int i = 0; i < (256 << litLogSize); i++) _probs[LITERAL_FLAG][i] = PSCALE >> 1; } void ROLZEncoder::encodeBits(int val, int n) { _c1 = 1; do { n--; encodeBit(val & (1 << n)); } while (n != 0); } void ROLZEncoder::encode9Bits(int val) { _c1 = 1; encodeBit(val & 0x100); encodeBit(val & 0x80); encodeBit(val & 0x40); encodeBit(val & 0x20); encodeBit(val & 0x10); encodeBit(val & 0x08); encodeBit(val & 0x04); encodeBit(val & 0x02); encodeBit(val & 0x01); } void ROLZEncoder::dispose() { for (int i = 0; i < 8; i++) { _buf[_idx + i] = kanzi::byte(_low >> 56); _low <<= 8; } _idx += 8; } const uint64 ROLZDecoder::TOP = 0x00FFFFFFFFFFFFFF; const uint64 ROLZDecoder::MASK_0_56 = 0x00FFFFFFFFFFFFFF; const uint64 ROLZDecoder::MASK_0_32 = 0x00000000FFFFFFFF; const int ROLZDecoder::MATCH_FLAG = 0; const int ROLZDecoder::LITERAL_FLAG = 1; const int ROLZDecoder::PSCALE = 0xFFFF; ROLZDecoder::ROLZDecoder(uint litLogSize, uint mLogSize, kanzi::byte buf[], int& idx, int end) : _idx(idx) , _low(0) , _high(TOP) , _current(0) , _buf(buf) , _end(end) , _error(false) , _c1(1) , _ctx(0) , _pIdx(LITERAL_FLAG) { if ((_idx < 0) || (_idx > _end) || (_end - _idx < 8)) { _error = true; } else { for (int i = 0; i < 8; i++) _current = (_current << 8) | (uint64(_buf[_idx + i]) & 0xFF); _idx += 8; } _logSizes[MATCH_FLAG] = mLogSize; _logSizes[LITERAL_FLAG] = litLogSize; _probs[MATCH_FLAG] = new uint16[256 << mLogSize]; _probs[LITERAL_FLAG] = new uint16[256 << litLogSize]; reset(); } void ROLZDecoder::reset() { const int mLogSize = _logSizes[MATCH_FLAG]; for (int i = 0; i < (256 << mLogSize); i++) _probs[MATCH_FLAG][i] = PSCALE >> 1; const int litLogSize = _logSizes[LITERAL_FLAG]; for (int i = 0; i < (256 << litLogSize); i++) _probs[LITERAL_FLAG][i] = PSCALE >> 1; } int ROLZDecoder::decodeBits(int n) { _c1 = 1; const int mask = (1 << n) - 1; do { decodeBit(); n--; } while (n != 0); return _c1 & mask; } int ROLZDecoder::decode9Bits() { _c1 = 1; decodeBit(); decodeBit(); decodeBit(); decodeBit(); decodeBit(); decodeBit(); decodeBit(); decodeBit(); decodeBit(); return _c1 & 0x1FF; } const int ROLZCodec2::MATCH_FLAG = 0; const int ROLZCodec2::LITERAL_FLAG = 1; const int ROLZCodec2::MATCH_CTX = 0; const int ROLZCodec2::LITERAL_CTX = 1; const int ROLZCodec2::MIN_MATCH3 = 3; const int ROLZCodec2::MIN_MATCH7 = 7; const int ROLZCodec2::MAX_MATCH = MIN_MATCH3 + 255; const int ROLZCodec2::LOG_POS_CHECKS = 5; ROLZCodec2::ROLZCodec2(uint logPosChecks) : _logPosChecks(logPosChecks) { if ((logPosChecks < 2) || (logPosChecks > 8)) { stringstream ss; ss << "ROLZX codec: Invalid logPosChecks parameter: " << logPosChecks << " (must be in [2..8])"; throw invalid_argument(ss.str()); } _pCtx = nullptr; _posChecks = 1 << _logPosChecks; _maskChecks = uint8(_posChecks - 1); _minMatch = MIN_MATCH3; _matches = new uint32[ROLZCodec::HASH_SIZE << _logPosChecks]; memset(&_counters[0], 0, sizeof(_counters)); } ROLZCodec2::ROLZCodec2(Context& ctx) : _pCtx(&ctx) { _logPosChecks = LOG_POS_CHECKS; _posChecks = 1 << _logPosChecks; _maskChecks = uint8(_posChecks - 1); _minMatch = MIN_MATCH3; _matches = new uint32[ROLZCodec::HASH_SIZE << _logPosChecks]; memset(&_counters[0], 0, sizeof(_counters)); } // return position index (_logPosChecks bits) + length (16 bits) or -1 int ROLZCodec2::findMatch(const kanzi::byte buf[], int pos, int end, uint32 key) { const int counter = _counters[key]; uint32* matches = &_matches[key << _logPosChecks]; prefetchRead(matches); const kanzi::byte* curBuf = &buf[pos]; const uint32 hash32 = ROLZCodec::hash(curBuf); int bestLen = 0; int bestIdx = -1; const int maxMatch = min(ROLZCodec2::MAX_MATCH, end - pos) - 8; // Check all recorded positions for (int i = counter; i > counter - _posChecks; i--) { uint32 ref = matches[i & _maskChecks]; // Hash check may save a memory access ... if ((ref & ROLZCodec::HASH_MASK) != hash32) continue; ref &= ~ROLZCodec::HASH_MASK; if (buf[ref + bestLen] != curBuf[bestLen]) continue; int n = 0; while (n < maxMatch) { const int64 diff = LittleEndian::readLong64(&buf[ref + n]) ^ LittleEndian::readLong64(&curBuf[n]); if (diff != 0) { n += (Global::trailingZeros(uint64(diff)) >> 3); break; } n += 8; } if (n > bestLen) { bestIdx = counter - i; bestLen = n; if (bestLen == maxMatch) break; } } // Register current position _counters[key] = (_counters[key] + 1) & _maskChecks; matches[_counters[key]] = hash32 | int32(pos); return (bestLen < _minMatch) ? -1 : (bestIdx << 16) | (bestLen - _minMatch); } bool ROLZCodec2::forward(SliceArray& input, SliceArray& output, int count) { const int outputIndex = output._index; const int outputCapacity = output._length - outputIndex; if (outputCapacity < getMaxEncodedLength(count)) return false; const int srcEnd = count - 4; const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[outputIndex]; BigEndian::writeInt32(&dst[0], count); _minMatch = MIN_MATCH3; int flags = 0; int delta = 2; if (_pCtx != nullptr) { Global::DataType dt = (Global::DataType) _pCtx->getInt("dataType", Global::UNDEFINED); if (dt == Global::UNDEFINED) { uint freqs0[256] = { 0 }; Global::computeHistogram(&src[0], count, freqs0); dt = Global::detectSimpleType(count, freqs0); if (dt != Global::UNDEFINED) _pCtx->putInt("dataType", dt); } if (dt == Global::EXE) { delta = 3; flags |= 8; } else if (dt == Global::DNA) { delta = 8; _minMatch = MIN_MATCH7; flags |= 4; } } const int dt = delta; const bool cond = _minMatch == MIN_MATCH3; dst[4] = kanzi::byte(flags); int srcIdx = 0; int dstIdx = 5; int sizeChunk = min(count, ROLZCodec::CHUNK_SIZE); int startChunk = 0; ROLZEncoder re(9, _logPosChecks, &dst[0], dstIdx); memset(&_counters[0], 0, sizeof(_counters)); while (startChunk < srcEnd) { memset(&_matches[0], 0, sizeof(uint32) * size_t(ROLZCodec::HASH_SIZE << _logPosChecks)); const int endChunk = min(startChunk + sizeChunk, srcEnd); sizeChunk = endChunk - startChunk; re.reset(); src = &input._array[startChunk]; srcIdx = 0; // First literals const int n = min(srcEnd - startChunk, 8); re.setContext(LITERAL_CTX, kanzi::byte(0)); for (int j = 0; j < n; j++) { re.encode9Bits((LITERAL_FLAG << 8) | int(src[srcIdx])); srcIdx++; } while (srcIdx < sizeChunk) { re.setContext(LITERAL_CTX, src[srcIdx - 1]); uint32 key = (cond == true) ? ROLZCodec::getKey1(&src[srcIdx - dt]) : ROLZCodec::getKey2(&src[srcIdx - dt]); const int match = findMatch(src, srcIdx, sizeChunk, key); if (match < 0) { // Emit one literal re.encode9Bits((LITERAL_FLAG << 8) | int(src[srcIdx])); srcIdx++; continue; } // Emit one match length and index const int matchLen = match & 0xFFFF; re.encode9Bits((MATCH_FLAG << 8) | matchLen); const int matchIdx = match >> 16; re.setContext(MATCH_CTX, src[srcIdx - 1]); re.encodeBits(matchIdx, _logPosChecks); srcIdx += (matchLen + _minMatch); } startChunk = endChunk; } // Emit last literals for (int i = 0; i < 4; i++, srcIdx++) { re.setContext(LITERAL_CTX, src[srcIdx - 1]); re.encode9Bits((LITERAL_FLAG << 8) | int(src[srcIdx])); } re.dispose(); input._index = startChunk - sizeChunk + srcIdx; output._index += dstIdx; return (input._index == count) && (dstIdx < count); } bool ROLZCodec2::inverse(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (input._array == output._array) return false; if ((count < 5) || (input._index < 0) || (input._length < 0) || (input._index > input._length - count)) return false; kanzi::byte* src = &input._array[input._index]; const int dstEnd = BigEndian::readInt32(&src[0]); const int outputCapacity = output._length - output._index; if ((dstEnd <= 0) || (dstEnd > outputCapacity)) return false; int srcIdx = 5; int sizeChunk = min(dstEnd, ROLZCodec::CHUNK_SIZE); int startChunk = 0; _minMatch = MIN_MATCH3; const int flags = int(src[4]); int delta = 2; if ((flags & 0x0E) == 8) { delta = 3; } else if ((flags & 0x0E) == 4) { delta = 8; _minMatch = MIN_MATCH7; } const bool cond = _minMatch == MIN_MATCH3; ROLZDecoder rd(9, _logPosChecks, &src[0], srcIdx, count); memset(&_counters[0], 0, sizeof(_counters)); while (startChunk < dstEnd) { memset(&_matches[0], 0, sizeof(uint32) * (ROLZCodec::HASH_SIZE << _logPosChecks)); const int endChunk = min(startChunk + sizeChunk, dstEnd); sizeChunk = endChunk - startChunk; if (startChunk > 0) rd.reset(); kanzi::byte* dst = &output._array[output._index]; kanzi::byte* refBuf = &output._array[output._index - delta]; int dstIdx = 0; // First literals rd.setContext(LITERAL_CTX, kanzi::byte(0)); const int n = min(sizeChunk, 8); for (int j = 0; j < n; j++) { int val = rd.decode9Bits(); if (rd.isValid() == false) { output._index += dstIdx; return false; } // Sanity check if ((val >> 8) == MATCH_FLAG) { output._index += dstIdx; return false; } dst[dstIdx++] = kanzi::byte(val); } // Next chunk while (dstIdx < sizeChunk) { const int savedIdx = dstIdx; const uint32 key = (cond == true) ? ROLZCodec::getKey1(&refBuf[dstIdx]) : ROLZCodec::getKey2(&refBuf[dstIdx]); uint32* matches = &_matches[key << _logPosChecks]; rd.setContext(LITERAL_CTX, dst[dstIdx - 1]); int val = rd.decode9Bits(); if (rd.isValid() == false) { output._index += dstIdx; return false; } if ((val >> 8) == LITERAL_FLAG) { dst[dstIdx++] = kanzi::byte(val); } else { // Read one match length and index const int matchLen = val & 0xFF; prefetchRead(&_counters[key]); const int copyLen = matchLen + _minMatch; // Sanity check against the remaining space in the current // chunk. The match length is stored without the minimum // match size, which is data type dependent. if (copyLen > sizeChunk - dstIdx) { output._index += dstIdx; return false; } rd.setContext(MATCH_CTX, dst[dstIdx - 1]); const int32 matchIdx = int32(rd.decodeBits(_logPosChecks)); if (rd.isValid() == false) { output._index += dstIdx; return false; } const int32 ref = matches[(_counters[key] - matchIdx) & _maskChecks]; // emitCopy() uses 8-byte stores when the match distance is // large and may write up to seven bytes past the logical end. // Check that allowance against the absolute output position; // dstIdx alone is relative to the current chunk. if ((dstIdx - ref >= 8) && (startChunk + dstIdx + copyLen > outputCapacity - 7)) { output._index += dstIdx; return false; } dstIdx = ROLZCodec::emitCopy(dst, dstIdx, ref, copyLen); } // Update map _counters[key]++; matches[_counters[key] & _maskChecks] = savedIdx; } startChunk = endChunk; output._index += dstIdx; } rd.dispose(); input._index = srcIdx; return srcIdx == count; } kanzi-cpp-2.6.0/src/transform/ROLZCodec.hpp000066400000000000000000000240341525576310500204660ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_ROLZCodec #define knz_ROLZCodec #include "../Context.hpp" #include "../Memory.hpp" #include "../Transform.hpp" // Implementation of a Reduced Offset Lempel Ziv transform // More information about ROLZ at http://ezcodesample.com/rolz/rolz_article.html namespace kanzi { class ROLZEncoder { private: static const uint64 TOP; static const uint64 MASK_0_32; static const int MATCH_FLAG; static const int LITERAL_FLAG; static const int PSCALE; uint16* _probs[2]; uint _logSizes[2]; int& _idx; uint64 _low; uint64 _high; byte* _buf; int32 _c1; int32 _ctx; int _pIdx; void encodeBit(int bit); public: ROLZEncoder(uint litLogSize, uint mLogSize, byte buf[], int& idx); ~ROLZEncoder() { delete[] _probs[LITERAL_FLAG]; delete[] _probs[MATCH_FLAG]; } void encodeBits(int val, int n); void encode9Bits(int val); void dispose(); void reset(); void setContext(int n, byte ctx) { _pIdx = n; _ctx = int32(ctx) << _logSizes[_pIdx]; } }; class ROLZDecoder { private: static const uint64 TOP; static const uint64 MASK_0_56; static const uint64 MASK_0_32; static const int MATCH_FLAG; static const int LITERAL_FLAG; static const int PSCALE; uint16* _probs[2]; uint _logSizes[2]; int& _idx; uint64 _low; uint64 _high; uint64 _current; byte* _buf; int _end; bool _error; int32 _c1; int32 _ctx; int _pIdx; int decodeBit(); public: ROLZDecoder(uint litLogSize, uint mLogSize, byte buf[], int& idx, int end); ~ROLZDecoder() { delete[] _probs[LITERAL_FLAG]; delete[] _probs[MATCH_FLAG]; } int decodeBits(int n); int decode9Bits(); void dispose() const {} void reset(); void setContext(int n, byte ctx) { _pIdx = n; _ctx = int32(ctx) << _logSizes[_pIdx]; } bool isValid() const { return _error == false; } }; // Use ANS to encode/decode literals and matches class ROLZCodec1 FINAL : public Transform { public: ROLZCodec1(uint logPosChecks); ROLZCodec1(Context& ctx); ~ROLZCodec1() { if (_matches != nullptr) delete[] _matches; } bool forward(SliceArray& src, SliceArray& dst, int length); bool inverse(SliceArray& src, SliceArray& dst, int length); // Required encoding output buffer size int getMaxEncodedLength(int srcLen) const { return (srcLen <= 512) ? srcLen + 64 : srcLen; } private: static const int MIN_MATCH3; static const int MIN_MATCH4; static const int MIN_MATCH7; static const int MAX_MATCH; static const int LOG_POS_CHECKS; uint32* _matches; size_t _mSize; uint8 _counters[65536]; int _logPosChecks; int _posChecks; Context* _pCtx; int _minMatch; uint8 _maskChecks; int findMatch(const byte buf[], int pos, int end, uint32 hash32, const uint32* matches, const uint8* counter) const; int emitLength(byte block[], int length) const; int readLength(const byte block[], int& idx) const; }; // Use CM (ROLZEncoder/ROLZDecoder) to encode/decode literals and matches // Code loosely based on 'balz' by Ilya Muravyov class ROLZCodec2 FINAL : public Transform { public: ROLZCodec2(uint logPosChecks); ROLZCodec2(Context& ctx); ~ROLZCodec2() { if (_matches != nullptr) delete[] _matches; } bool forward(SliceArray& src, SliceArray& dst, int length); bool inverse(SliceArray& src, SliceArray& dst, int length); // Required encoding output buffer size int getMaxEncodedLength(int srcLen) const { // Since we do not check the dst index for each byte (for speed purpose) // allocate some extra buffer for incompressible data. return srcLen + ((srcLen < 32768) ? 1024 : srcLen >> 5); } private: static const int MATCH_FLAG; static const int LITERAL_FLAG; static const int MATCH_CTX; static const int LITERAL_CTX; static const int MIN_MATCH3; static const int MIN_MATCH7; static const int MAX_MATCH; static const int LOG_POS_CHECKS; uint32* _matches; uint8 _counters[65536]; int _logPosChecks; uint8 _maskChecks; Context* _pCtx; int _minMatch; int _posChecks; int findMatch(const byte buf[], int pos, int end, uint32 key); }; class ROLZCodec FINAL : public Transform { friend class ROLZCodec1; friend class ROLZCodec2; public: ROLZCodec(uint logPosChecks = 4); ROLZCodec(Context& ctx); ~ROLZCodec() { delete _delegate; } bool forward(SliceArray& src, SliceArray& dst, int length); bool inverse(SliceArray& src, SliceArray& dst, int length); // Required encoding output buffer size int getMaxEncodedLength(int srcLen) const { return _delegate->getMaxEncodedLength(srcLen); } private: static const int HASH_SIZE; static const int CHUNK_SIZE; static const int32 HASH; static const int32 HASH_MASK; static const int MAX_BLOCK_SIZE; static const int MIN_BLOCK_SIZE; Transform* _delegate; static uint32 getKey1(const byte* p) { return uint32(LittleEndian::readInt16(p)) & (HASH_SIZE - 1); } static uint32 getKey2(const byte* p) { return uint32((uint64(LittleEndian::readLong64(p)) * HASH) >> 40) & (HASH_SIZE - 1); } static uint32 hash(const byte* p) { return ((uint32(LittleEndian::readInt32(p)) << 8) * HASH) & HASH_MASK; } static int emitCopy(byte dst[], int dstIdx, int ref, int matchLen); }; inline int ROLZCodec1::emitLength(byte block[], int length) const { if (length < 1 << 7) { block[0] = byte(length); return 1; } int idx = 0; if (length >= 1 << 14) { block[idx] = byte(0x80 | (length >> 21)); idx += ((length >= 1 << 21) ? 1 : 0); block[idx++] = byte(0x80 | (length >> 14)); } block[idx++] = byte(0x80 | (length >> 7)); block[idx++] = byte(length & 0x7F); return idx; } inline int ROLZCodec1::readLength(const byte block[], int& pos) const { int next = int(block[pos++]); if (next < 128) return next; int length = next & 0x7F; next = int(block[pos++]); length = (length << 7) | (next & 0x7F); if (next >= 128) { next = int(block[pos++]); length = (length << 7) | (next & 0x7F); if (next >= 128) { next = int(block[pos++]); length = (length << 7) | (next & 0x7F); } } return length; } inline int ROLZCodec::emitCopy(byte buf[], int dstIdx, int ref, int matchLen) { const int res = dstIdx + matchLen; if (dstIdx - ref >= 8) { while (matchLen > 0) { KANZI_MEM_CP8(&buf[dstIdx], &buf[ref]); ref += 8; dstIdx += 8; matchLen -= 8; } } else { while (matchLen != 0) { buf[dstIdx++] = buf[ref++]; matchLen--; } } return res; } inline void ROLZEncoder::encodeBit(int bit) { const uint64 split = ((_high - _low) >> 4) * uint64(_probs[_pIdx][_ctx + _c1] >> 4) >> 8; // Update fields with new interval bounds if (bit == 0) { _low += (split + 1); _probs[_pIdx][_ctx + _c1] -= (_probs[_pIdx][_ctx + _c1] >> 5); _c1 += _c1; } else { _high = _low + split; _probs[_pIdx][_ctx + _c1] -= ((_probs[_pIdx][_ctx + _c1] - PSCALE + 32) >> 5); _c1 += (_c1 + 1); } // Emit unchanged first 32 bits while (((_low ^ _high) >> 24) == 0) { BigEndian::writeInt32(&_buf[_idx], int32(_high >> 32)); _idx += 4; _low <<= 32; _high = (_high << 32) | MASK_0_32; } } inline int ROLZDecoder::decodeBit() { if (_error) return 0; const uint64 mid = _low + (((_high - _low) >> 4) * uint64(_probs[_pIdx][_ctx + _c1] >> 4) >> 8); int bit; // Update bounds and predictor if (mid >= _current) { bit = 1; _high = mid; _probs[_pIdx][_ctx + _c1] -= ((_probs[_pIdx][_ctx + _c1] - PSCALE + 32) >> 5); _c1 += (_c1 + 1); } else { bit = 0; _low = mid + 1; _probs[_pIdx][_ctx + _c1] -= (_probs[_pIdx][_ctx + _c1] >> 5); _c1 += _c1; } // Read 32 bits while (((_low ^ _high) >> 24) == 0) { if ((_idx < 0) || (_idx > _end) || (_end - _idx < 4)) { _error = true; return bit; } _low = (_low << 32) & MASK_0_56; _high = ((_high << 32) | MASK_0_32) & MASK_0_56; const uint64 val = uint64(BigEndian::readInt32(&_buf[_idx])) & MASK_0_32; _current = ((_current << 32) | val) & MASK_0_56; _idx += 4; } return bit; } } #endif kanzi-cpp-2.6.0/src/transform/SBRT.cpp000066400000000000000000000101031525576310500174770ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include "SBRT.hpp" using namespace kanzi; const int SBRT::MODE_MTF = 1; // alpha = 0 const int SBRT::MODE_RANK = 2; // alpha = 1/2 const int SBRT::MODE_TIMESTAMP = 3; // alpha = 1 SBRT::SBRT(int mode) : _mask1((mode == MODE_TIMESTAMP) ? 0 : -1) , _mask2((mode == MODE_MTF) ? 0 : -1) , _shift((mode == MODE_RANK) ? 1 : 0) { if ((mode != MODE_MTF) && (mode != MODE_RANK) && (mode != MODE_TIMESTAMP)) throw std::invalid_argument("Invalid mode parameter"); } SBRT::SBRT(int mode, Context&) : _mask1((mode == MODE_TIMESTAMP) ? 0 : -1) , _mask2((mode == MODE_MTF) ? 0 : -1) , _shift((mode == MODE_RANK) ? 1 : 0) { if ((mode != MODE_MTF) && (mode != MODE_RANK) && (mode != MODE_TIMESTAMP)) throw std::invalid_argument("Invalid mode parameter"); } bool SBRT::forward(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (!SliceArray::isValid(input)) throw std::invalid_argument("SBRT: Invalid input block"); if (!SliceArray::isValid(output)) throw std::invalid_argument("SBRT: Invalid output block"); if ((count < 0) || (count > input._length - input._index) || (count > output._length - output._index)) return false; // Aliasing const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; int p[256] = { 0 }; int q[256] = { 0 }; uint8 s2r[256]; uint8 r2s[256]; for (int i = 0; i < 256; i++) { s2r[i] = uint8(i); r2s[i] = uint8(i); } for (int i = 0; i < count; i++) { const uint8 c = uint8(src[i]); int r = int(s2r[c]); dst[i] = kanzi::byte(r); const int qc = ((i & _mask1) + (p[c] & _mask2)) >> _shift; p[c] = i; q[c] = qc; // Move up symbol to correct rank while ((r > 0) && (q[r2s[r - 1]] <= qc)) { r2s[r] = r2s[r - 1]; s2r[r2s[r]] = uint8(r); r--; } r2s[r] = c; s2r[c] = uint8(r); } input._index += count; output._index += count; return true; } bool SBRT::inverse(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (!SliceArray::isValid(input)) throw std::invalid_argument("SBRT: Invalid input block"); if (!SliceArray::isValid(output)) throw std::invalid_argument("SBRT: Invalid output block"); int p[256] = { 0 }; int q[256] = { 0 }; uint8 r2s[256]; for (int i = 0; i < 256; i++) r2s[i] = uint8(i); // Check before forming slice pointers. This placement also keeps the inverse loop fast. if ((count < 0) || (count > input._length - input._index) || (count > output._length - output._index)) return false; // Aliasing const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; for (int i = 0; i < count; i++) { int r = int(src[i]); const int c = int(r2s[r]); dst[i] = kanzi::byte(r2s[r]); const int qc = ((i & _mask1) + (p[c] & _mask2)) >> _shift; p[c] = i; q[c] = qc; // Move up symbol to correct rank while ((r > 0) && (q[r2s[r - 1]] <= qc)) { r2s[r] = r2s[r - 1]; r--; } r2s[r] = uint8(c); } input._index += count; output._index += count; return true; } kanzi-cpp-2.6.0/src/transform/SBRT.hpp000066400000000000000000000036011525576310500175110ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_SBRT #define knz_SBRT #include "../Context.hpp" #include "../Transform.hpp" namespace kanzi { // Sort by Rank Transform is a family of transforms typically used after // a BWT to reduce the variance of the data prior to entropy coding. // SBR(alpha) is defined by sbr(x, alpha) = (1-alpha)*(t-w1(x,t)) + alpha*(t-w2(x,t)) // where x is an item in the data list, t is the current access time and wk(x,t) is // the k-th access time to x at time t (with 0 <= alpha <= 1). // See [Two new families of list update algorithms] by Frank Schulz for details. // SBR(0)= Move to Front Transform // SBR(1)= Time Stamp Transform // This code implements SBR(0), SBR(1/2) and SBR(1). Code derived from openBWT class SBRT FINAL : public Transform { public: static const int MODE_MTF; static const int MODE_RANK; static const int MODE_TIMESTAMP; SBRT(int mode); SBRT(int mode, Context&); ~SBRT() {} bool forward(SliceArray& input, SliceArray& output, int length); bool inverse(SliceArray& input, SliceArray& output, int length); int getMaxEncodedLength(int srcLen) const { return srcLen; } private: const int _mask1; const int _mask2; const int _shift; }; } #endif kanzi-cpp-2.6.0/src/transform/SRT.cpp000066400000000000000000000173121525576310500174060ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include "SRT.hpp" using namespace kanzi; bool SRT::forward(SliceArray& input, SliceArray& output, int length) { if (length == 0) return true; if (!SliceArray::isValid(input)) throw std::invalid_argument("SRT: Invalid input block"); if (!SliceArray::isValid(output)) throw std::invalid_argument("SRT: Invalid output block"); if (output._length - output._index < getMaxEncodedLength(length)) return false; uint freqs[256] = { 0 }; uint8 s2r[256] = { 0 }; uint8 r2s[256] = { 0 }; const kanzi::byte* src = &input._array[input._index]; // find first symbols and count occurrences for (int i = 0, b = 0; i < length;) { uint8 c = uint8(src[i]); int j = i + 1; while ((j < length) && (src[j] == kanzi::byte(c))) j++; if (freqs[c] == 0) { r2s[b] = c; s2r[c] = uint8(b); b++; } freqs[c] += (j - i); i = j; } // init arrays uint8 symbols[256]; int buckets[256] = { 0 }; const int nbSymbols = preprocess(freqs, symbols); for (int i = 0, bucketPos = 0; i < nbSymbols; i++) { const uint8 c = symbols[i]; buckets[c] = bucketPos; bucketPos += freqs[c]; } const int headerSize = encodeHeader(freqs, &output._array[output._index]); output._index += headerSize; kanzi::byte* dst = &output._array[output._index]; // encoding for (int i = 0; i < length;) { uint8 c = uint8(src[i]); int r = s2r[c]; int p = buckets[c]; dst[p] = kanzi::byte(r); p++; if (r != 0) { do { const uint8 t = r2s[r - 1]; r2s[r] = t; s2r[t] = uint8(r); r--; } while (r != 0); r2s[0] = c; s2r[c] = 0; } i++; while ((i < length) && (src[i] == kanzi::byte(c))) { dst[p] = kanzi::byte(0); p++; i++; } buckets[c] = p; } input._index += length; output._index += length; return true; } bool SRT::inverse(SliceArray& input, SliceArray& output, int length) { if (length == 0) return true; if (!SliceArray::isValid(input)) throw std::invalid_argument("SRT: Invalid input block"); if (!SliceArray::isValid(output)) throw std::invalid_argument("SRT: Invalid output block"); if ((length < 256) || (input._index + length > input._length)) return false; uint freqs[256] = { 0 }; const int headerSize = decodeHeader(&input._array[input._index], length, freqs); if (headerSize < 0) return false; const int dataLength = length - headerSize; if (dataLength < 0) return false; // The frequency table must describe exactly the data following the header. // Otherwise a bucket end can point past the input block and the decoding // loop below can read outside the transform payload. uint64 totalFreq = 0; for (int i = 0; i < 256; i++) totalFreq += freqs[i]; if (totalFreq != uint64(dataLength)) return false; input._index += headerSize; length = dataLength; if (length > output._length - output._index) return false; const kanzi::byte* src = &input._array[input._index]; uint8 symbols[256] = { 0 }; // init arrays int nbSymbols = preprocess(freqs, symbols); int buckets[256] = { 0 }; int bucketEnds[256] = { 0 }; uint8 r2s[256] = { 0 }; for (int i = 0, bucketPos = 0; i < nbSymbols; i++) { const uint8 c = symbols[i]; if ((bucketPos < 0) || (bucketPos >= length)) return false; r2s[int(src[bucketPos])] = c; buckets[c] = bucketPos + 1; bucketPos += freqs[c]; bucketEnds[c] = bucketPos; } uint8 c = r2s[0]; kanzi::byte* dst = &output._array[output._index]; // decoding for (int i = 0; i < length; i++) { dst[i] = kanzi::byte(c); if (buckets[c] < bucketEnds[c]) { const uint8 r = uint8(src[buckets[c]]); buckets[c]++; if (r == 0) continue; if (r <= 8) { if (r >= 1) r2s[0] = r2s[1]; if (r >= 2) r2s[1] = r2s[2]; if (r >= 3) r2s[2] = r2s[3]; if (r >= 4) r2s[3] = r2s[4]; if (r >= 5) r2s[4] = r2s[5]; if (r >= 6) r2s[5] = r2s[6]; if (r >= 7) r2s[6] = r2s[7]; if (r >= 8) r2s[7] = r2s[8]; } else { memmove(&r2s[0], &r2s[1], size_t(r)); } r2s[r] = c; c = r2s[0]; } else { if (nbSymbols == 1) continue; nbSymbols--; memmove(&r2s[0], &r2s[1], size_t(nbSymbols)); c = r2s[0]; } } input._index += length; output._index += length; return true; } int SRT::preprocess(const uint freqs[], uint8 symbols[]) { int nbSymbols = 0; for (int i = 0; i < 256; i++) { if (freqs[i] == 0) continue; symbols[nbSymbols] = uint8(i); nbSymbols++; } int h = 4; while (h < nbSymbols) h = h * 3 + 1; do { h /= 3; for (int i = h; i < nbSymbols; i++) { uint8 t = symbols[i]; int b; for (b = i - h; b >= 0; b -= h) { const int val = freqs[symbols[b]] - freqs[t]; if ((val >= 0) && ((val != 0) || (t >= symbols[b]))) break; symbols[b + h] = symbols[b]; } symbols[b + h] = t; } } while (h != 1); return nbSymbols; } int SRT::encodeHeader(const uint freqs[], kanzi::byte dst[]) { int dstIdx = 0; for (int i = 0; i < 256; i++) { uint f = freqs[i]; if (f >= 128) { dst[dstIdx++] = kanzi::byte(0x80 | f); f >>= 7; if (f >= 128) { dst[dstIdx++] = kanzi::byte(0x80 | f); f >>= 7; if (f >= 128) { dst[dstIdx++] = kanzi::byte(0x80 | f); f >>= 7; if (f >= 128) { dst[dstIdx++] = kanzi::byte(0x80 | f); f >>= 7; } } } } dst[dstIdx++] = kanzi::byte(f); } return dstIdx; } int SRT::decodeHeader(const kanzi::byte src[], int srcEnd, uint freqs[]) { int srcIdx = 0; for (int i = 0; i < 256; i++) { uint res = 0; int shift = 0; // Frequencies are encoded as varints with up to 5 bytes. for (int j = 0; j < 5; j++) { if (srcIdx >= srcEnd) return -1; const uint val = uint(src[srcIdx++]); res |= ((val & 0x7F) << shift); if ((val & 0x80) == 0) break; if (j == 4) return -1; shift += 7; } freqs[i] = res; } return srcIdx; } kanzi-cpp-2.6.0/src/transform/SRT.hpp000066400000000000000000000026361525576310500174160ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_SRT #define knz_SRT #include "../Context.hpp" #include "../Transform.hpp" namespace kanzi { // Sorted Rank Transform is typically used after a BWT to reduce the variance // of the data prior to entropy coding. class SRT FINAL : public Transform { public: SRT() {} SRT(Context&) {} ~SRT() {} bool forward(SliceArray& pSrc, SliceArray& pDst, int length); bool inverse(SliceArray& pSrc, SliceArray& pDst, int length); int getMaxEncodedLength(int srcLen) const { return srcLen + 1024 /* max header size */; } private: static int preprocess(const uint freqs[], uint8 symbols[]); static int encodeHeader(const uint freqs[], byte dst[]); static int decodeHeader(const byte src[], int srcEnd, uint freqs[]); }; } #endif kanzi-cpp-2.6.0/src/transform/TextCodec.cpp000066400000000000000000001616101525576310500206210ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include "TextCodec.hpp" #include "../Global.hpp" #include "../Magic.hpp" using namespace kanzi; using namespace std; const int TextCodec::MAX_DICT_SIZE = 1 << 19; // must be less than 1<<24 const int TextCodec::MAX_WORD_LENGTH = 31; // must be less than 128 const int TextCodec::MIN_BLOCK_SIZE = 1024; const int TextCodec::MAX_BLOCK_SIZE = 1 << 30; // 1 GB const kanzi::byte TextCodec::ESCAPE_TOKEN1 = kanzi::byte(0x0F); // dictionary word preceded by space symbol const kanzi::byte TextCodec::ESCAPE_TOKEN2 = kanzi::byte(0x0E); // toggle upper/lower case of first word char const kanzi::byte TextCodec::MASK_1F = kanzi::byte(0x1F); const kanzi::byte TextCodec::MASK_3F = kanzi::byte(0x3F); const kanzi::byte TextCodec::MASK_20 = kanzi::byte(0x20); const kanzi::byte TextCodec::MASK_40 = kanzi::byte(0x40); const kanzi::byte TextCodec::MASK_80 = kanzi::byte(0x80); const kanzi::byte TextCodec::MASK_FLIP_CASE = kanzi::byte(0x80); const int TextCodec::HASH1 = 0x7FEB352D; const int TextCodec::HASH2 = 0x846CA68B; const kanzi::byte TextCodec::CR = kanzi::byte(0x0D); const kanzi::byte TextCodec::LF = kanzi::byte(0x0A); const kanzi::byte TextCodec::SP = kanzi::byte(0x20); const int TextCodec::THRESHOLD1 = 128; const int TextCodec::THRESHOLD2 = TextCodec::THRESHOLD1 * TextCodec::THRESHOLD1; const int TextCodec::THRESHOLD3 = 64; const int TextCodec::THRESHOLD4 = TextCodec::THRESHOLD3 * 128; const int TextCodec::LOG_HASHES_SIZE = 24; // 16 MB const kanzi::byte TextCodec::MASK_NOT_TEXT = kanzi::byte(0x80); const kanzi::byte TextCodec::MASK_CRLF = kanzi::byte(0x40); const kanzi::byte TextCodec::MASK_XML_HTML = kanzi::byte(0x20); const kanzi::byte TextCodec::MASK_TEXT_CODEC = kanzi::byte(0x10); const kanzi::byte TextCodec::MASK_DT = kanzi::byte(0x0F); const int TextCodec::MASK_LENGTH = 0x0007FFFF; // 19-bit dictionary index // V7 ranked dictionary indexes. The one-byte form has 63 usable values // because 0x80 is reserved for the case-flip marker. The two-byte form has // 8192 values, so the three-byte form starts at index 63 + 8192. static const int V7_INDEX_BASE2 = 63; static const int V7_INDEX_BASE3 = 8255; // 1024 of the most common English words with at least 2 chars. char TextCodec::DICT_EN_1024[] = "TheBeAndOfInToWithItThatForYouHeHaveOnSaidSayAtButWeByHadTheyAsW\ ouldWhoOrCanMayDoThisWasIsMuchAnyFromNotSheWhatTheirWhichGetGive\ HasAreHimHerComeMyOurWereWillSomeBecauseThereThroughTellWhenWork\ ThemYetUpOwnOutIntoJustCouldOverOldThinkDayWayThanLikeOtherHowTh\ enItsPeopleTwoMoreTheseBeenNowWantFirstNewUseSeeTimeManManyThing\ MakeHereWellOnlyHisVeryAfterWithoutAnotherNoAllBelieveBeforeOffT\ houghSoAgainstWhileLastTooDownTodaySameBackTakeEachDifferentWher\ eBetweenThoseEvenSeenUnderAboutOneAlsoFactMustActuallyPreventExp\ ectContainConcernIfSchoolYearGoingCannotDueEverTowardGirlFirmGla\ ssGasKeepWorldStillWentShouldSpendStageDoctorMightJobGoContinueE\ veryoneNeverAnswerFewMeanDifferenceTendNeedLeaveTryNiceHoldSomet\ hingAskWarmLipCoverIssueHappenTurnLookSureDiscoverFightMadDirect\ ionAgreeSomeoneFailRespectNoticeChoiceBeginThreeSystemLevelFeelM\ eetCompanyBoxShowPlayLiveLetterEggNumberOpenProblemFatHandMeasur\ eQuestionCallRememberCertainPutNextChairStartRunRaiseGoalReallyH\ omeTeaCandidateMoneyBusinessYoungGoodCourtFindKnowKindHelpNightC\ hildLotYourUsEyeYesWordBitVanMonthHalfLowMillionHighOrganization\ RedGreenBlueWhiteBlackYourselfEightBothLittleHouseLetDespiteProv\ ideServiceHimselfFriendDescribeFatherDevelopmentAwayKillTripHour\ GameOftenPlantPlaceEndAmongSinceStandDesignParticularSuddenlyMem\ berPayLawBookSilenceAlmostIncludeAgainEitherToolFourOnceLeastExp\ lainIdentifyUntilSiteMinuteCoupleWeekMatterBringDetailInformatio\ nNothingAnythingEverythingAgoLeadSometimesUnderstandWhetherNatur\ eTogetherFollowParentStopIndeedDifficultPublicAlreadySpeakMainta\ inRemainHearAllowMediaOfficeBenefitDoorHugPersonLaterDuringWarHi\ storyArgueWithinSetArticleStationMorningWalkEventWinChooseBehavi\ orShootFireFoodTitleAroundAirTeacherGapSubjectEnoughProveAcrossA\ lthoughHeadFootSecondBoyMainLieAbleCivilTableLoveProcessOfferStu\ dentConsiderAppearStudyBuyNearlyHumanEvidenceTextMethodIncluding\ SendRealizeSenseBuildControlAudienceSeveralCutCollegeInterestSuc\ cessSpecialRiskExperienceBehindBetterResultTreatFiveRelationship\ AnimalImproveHairStayTopReducePerhapsLateWriterPickElseSignifica\ ntChanceHotelGeneralRockRequireAlongFitThemselvesReportCondition\ ReachTruthEffortDecideRateEducationForceGardenDrugLeaderVoiceQui\ teWholeSeemMindFinallySirReturnFreeStoryRespondPushAccordingBrot\ herLearnSonHopeDevelopFeelingReadCarryDiseaseRoadVariousBallCase\ OperationCloseVisitReceiveBuildingValueResearchFullModelJoinSeas\ onKnownDirectorPositionPlayerSportErrorRecordRowDataPaperTheoryS\ paceEveryFormSupportActionOfficialWhoseIdeaHappyHeartBestTeamPro\ jectHitBaseRepresentTownPullBusMapDryMomCatDadRoomSmileFieldImpa\ ctFundLargeDogHugePrepareEnvironmentalProduceHerselfTeachOilSuch\ SituationTieCostIndustrySkinStreetImageItselfPhonePriceWearMostS\ unSoonClearPracticePieceWaitRecentImportantProductLeftWallSeries\ NewsShareMovieKidNorSimplyWifeOntoCatchMyselfFineComputerSongAtt\ entionDrawFilmRepublicanSecurityScoreTestStockPositiveCauseCentu\ ryWindowMemoryExistListenStraightCultureBillionFormerDecisionEne\ rgyMoveSummerWonderRelateAvailableLineLikelyOutsideShotShortCoun\ tryRoleAreaSingleRuleDaughterMarketIndicatePresentLandCampaignMa\ terialPopulationEconomyMedicalHospitalChurchGroundThousandAuthor\ ityInsteadRecentlyFutureWrongInvolveLifeHeightIncreaseRightBankC\ ulturalCertainlyWestExecutiveBoardSeekLongOfficerStatementRestBa\ yDealWorkerResourceThrowForwardPolicyScienceEyesBedItemWeaponFil\ lPlanMilitaryGunHotHeatAddressColdFocusForeignTreatmentBloodUpon\ CourseThirdWatchAffectEarlyStoreThusSoundEverywhereBabyAdministr\ ationMouthPageEnterProbablyPointSeatNaturalRaceFarChallengePassA\ pplyMailUsuallyMixToughClearlyGrowFactorStateLocalGuyEastSaveSou\ thSceneMotherCareerQuicklyCentralFaceIceAboveBeyondPictureNetwor\ kManagementIndividualWomanSizeSpeedBusySeriousOccurAddReadySignC\ ollectionListApproachChargeQualityPressureVoteNotePartRealWebCur\ rentDetermineTrueSadWhateverBreakWorryCupParticularlyAmountAbili\ tyEatRecognizeSitCharacterSomebodyLossDegreeEffectAttackStaffMid\ dleTelevisionWhyLegalCapitalTradeElectionEverybodyDropMajorViewS\ tandardBillEmployeeDiscussionOpportunityAnalysisTenSuggestLawyer\ HusbandSectionBecomeSkillSisterStyleCrimeProgramCompareCapMissBa\ dSortTrainingEasyNearRegionStrategyPurposePerformTechnologyEcono\ micBudgetExampleCheckEnvironmentDoneDarkTermRatherLaughGuessCarL\ owerHangPastSocialForgetHundredRemoveManagerEnjoyExactlyDieFinal\ MaybeHealthFloorChangeAmericanPoorFunEstablishTrialSpringDinnerB\ igThankProtectAvoidImagineTonightStarArmFinishMusicOwnerCryArtPr\ ivateOthersSimplePopularReflectEspeciallySmallLightMessageStepKe\ yPeaceProgressMadeSideGreatFixInterviewManageNationalFishLoseCam\ eraDiscussEqualWeightPerformanceSevenWaterProductionPersonalCell\ PowerEveningColorInsideBarUnitLessAdultWideRangeMentionDeepEdgeS\ trongHardTroubleNecessarySafeCommonFearFamilySeaDreamConferenceR\ eplyPropertyMeetingAlwaysStuffAgencyDeathGrowthSellSoldierActHea\ vyWetBagMarriageDeadSingRiseDecadeWhomFigurePoliceBodyMachineCat\ egoryAheadFrontCareOrderRealityPartnerYardBeatViolenceTotalDefen\ seWriteConsumerCenterGroupThoughtModernTaskCoachReasonAgeFingerS\ pecificConnectionWishResponsePrettyMovementCardLogNumberSumTreeE\ ntireCitizenThroughoutPetSimilarVictimNewspaperThreatClassShakeS\ ourceAccountPainFallRichPossibleAcceptSolidTravelTalkSaidCreateN\ onePlentyPeriodDefineNormalRevealDrinkAuthorServeNameMomentAgent\ DocumentActivityAnywayAfraidTypeActiveTrainInterestingRadioDange\ rGenerationLeafCopyMatchClaimAnyoneSoftwarePartyDeviceCodeLangua\ geLinkHoweverConfirmCommentCityAnywhereSomewhereDebateDriveHighe\ rBeautifulOnlineFanPriorityTraditionalSixUnited"; DictEntry TextCodec::STATIC_DICTIONARY[1024] = {}; int8 TextCodec::CHAR_TYPE[256] = {}; const bool TextCodec::INIT = TextCodec::init(TextCodec::CHAR_TYPE); const int TextCodec::STATIC_DICT_WORDS = TextCodec::createDictionary(DICT_EN_1024, sizeof(DICT_EN_1024), STATIC_DICTIONARY, 1024, 0); bool TextCodec::init(int8 cType[256]) { for (int i = 0; i < 256; i++) { if ((i >= ' ') && (i <= '/')) // [ !"#$%&'()*+,-./] cType[i] = 1; else if ((i >= ':') && (i <= '?')) // [:;<=>?] cType[i] = 1; else { switch (i) { case '\n': case '\r': case '\t': case '_': case '|': case '{': case '}': case '[': case ']': cType[i] = 1; break; default: cType[i] = (isUpperCase(byte(i)) || isLowerCase(byte(i))) == true ? 0 : -1; } } } return true; } // Create dictionary from array of words int TextCodec::createDictionary(char words[], int dictSize, DictEntry dict[], int maxWords, int startWord) { int delimAnchor = 0; int nbWords = startWord; byte* src = reinterpret_cast(words); for (int i = 0; ((i < dictSize) && (nbWords < maxWords)); i++) { if (isText(src[i]) == false) continue; if (isUpperCase(src[i])) { if (i > delimAnchor) { const int length = i - delimAnchor; const uint h = computeWordHash(&src[delimAnchor], length); dict[nbWords] = DictEntry(&src[delimAnchor], h, nbWords, length); nbWords++; delimAnchor = i; } src[i] ^= byte(0x20); } } if (nbWords < maxWords) { const int length = dictSize - 1 - delimAnchor; const uint h = computeWordHash(&src[delimAnchor], length); dict[nbWords] = DictEntry(&src[delimAnchor], h, nbWords, length); nbWords++; } return nbWords; } // Analyze the block and return an 8-bit status (see MASK flags constants) // The goal is to detect text data amenable to pre-processing. byte TextCodec::computeStats(const byte block[], int count, uint freqs0[], bool strict) { if ((strict == false) && (Magic::getType(block) != Magic::NO_MAGIC)) { // This is going to fail if the block is not the first of the file. // But this is a cheap test, good enough for fast mode. return TextCodec::MASK_NOT_TEXT; } uint* freqs1 = new uint[65536]; memset(&freqs1[0], 0, 65536 * sizeof(uint)); uint f0[256] = { 0 }; uint f1[256] = { 0 }; uint f3[256] = { 0 }; uint f2[256] = { 0 }; uint8 prv = 0; const uint8* data = reinterpret_cast(&block[0]); const int count4 = count & -4; // Unroll loop for (int i = 0; i < count4; i += 4) { const uint8 cur0 = data[i]; const uint8 cur1 = data[i + 1]; const uint8 cur2 = data[i + 2]; const uint8 cur3 = data[i + 3]; f0[cur0]++; f1[cur1]++; f2[cur2]++; f3[cur3]++; freqs1[(prv * 256) + cur0]++; freqs1[(cur0 * 256) + cur1]++; freqs1[(cur1 * 256) + cur2]++; freqs1[(cur2 * 256) + cur3]++; prv = cur3; } for (int i = count4; i < count; i++) { freqs0[data[i]]++; freqs1[(prv * 256) + data[i]]++; prv = data[i]; } for (int i = 0; i < 256; i++) { freqs0[i] += (f0[i] + f1[i] + f2[i] + f3[i]); } // Reject simple alphabets before the text heuristic. Ignore line // whitespace so wrapped DNA, Base64, and numeric data are detected too. uint freqsSimple[256]; memcpy(&freqsSimple[0], &freqs0[0], sizeof(freqsSimple)); const int nbWhitespace = freqsSimple[' '] + freqsSimple['\t'] + freqsSimple['\n'] + freqsSimple['\r']; const int simpleCount = count - nbWhitespace; freqsSimple[' '] = 0; freqsSimple['\t'] = 0; freqsSimple['\n'] = 0; freqsSimple['\r'] = 0; const Global::DataType simpleType = Global::detectSimpleType(simpleCount, freqsSimple); if (simpleType != Global::UNDEFINED) { delete[] freqs1; return TextCodec::MASK_NOT_TEXT | byte(simpleType); } const int cr = int(CR); const int lf = int(LF); int nbTextChars = freqs0[cr] + freqs0[lf]; int nbASCII = 0; for (int i = 0; i < 128; i++) { if (isText(byte(i)) == true) nbTextChars += freqs0[i]; nbASCII += freqs0[i]; } // Not text (crude thresholds) const int nbBinChars = count - nbASCII; bool notText = nbBinChars > (count >> 2); if (notText == false) { notText = nbTextChars < (count >> 2); if (strict == true) { notText |= ((freqs0[0] >= uint(count / 100)) || ((nbASCII / 95) < (count / 100))); } else { notText |= (freqs0[32] < uint(count / 50)); } } byte res = byte(0); if (notText == true) { res = detectType(freqs0, freqs1, count); delete[] freqs1; return res; } if (nbBinChars <= count - count / 10) { // Check if likely XML/HTML // Another crude test: check that the frequencies of < and > are similar // and 'high enough'. Also check it is worth to attempt replacing ampersand sequences. // Getting this flag wrong results in a very small compression speed degradation. const int f60 = freqs0[60]; // '<' const int f62 = freqs0[62]; // '>' const int f38 = freqs1[38 * 256 + 97] + freqs1[38 * 256 + 103] + freqs1[38 * 256 + 108] + freqs1[38 * 256 + 113]; // '&a', '&g', '&l', '&q' const int minFreq = max((count - nbBinChars) >> 9, 2); if ((f60 >= minFreq) && (f62 >= minFreq) && (f38 > 0)) { if (f60 < f62) { if (f60 >= (f62 - f62 / 100)) res |= TextCodec::MASK_XML_HTML; } else if (f62 < f60) { if (f62 >= (f60 - f60 / 100)) res |= TextCodec::MASK_XML_HTML; } else { res |= TextCodec::MASK_XML_HTML; } } } // Check CR+LF matches if ((freqs0[cr] != 0) && (freqs0[cr] == freqs0[lf])) { res |= TextCodec::MASK_CRLF; for (int i = 0; i < 256; i++) { if ((i != lf) && (freqs1[(cr * 256) + i]) != 0) { res &= ~TextCodec::MASK_CRLF; break; } if ((i != cr) && (freqs1[(i * 256) + lf]) != 0) { res &= ~TextCodec::MASK_CRLF; break; } } } delete[] freqs1; return res; } byte TextCodec::detectType(const uint freqs0[], const uint freqs1[], int count) { Global::DataType dt = Global::detectSimpleType(count, freqs0); if (dt != Global::UNDEFINED) return TextCodec::MASK_NOT_TEXT | byte(dt); // Valid UTF-8 sequences // See Unicode 16 Standard - UTF-8 Table 3.7 // U+0000..U+007F 00..7F // U+0080..U+07FF C2..DF 80..BF // U+0800..U+0FFF E0 A0..BF 80..BF // U+1000..U+CFFF E1..EC 80..BF 80..BF // U+D000..U+D7FF ED 80..9F 80..BF 80..BF // U+E000..U+FFFF EE..EF 80..BF 80..BF // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF // Check rules for 1 byte uint sum = freqs0[0xC0] + freqs0[0xC1]; uint sum2 = 0; bool res = true; for (int i = 0xF5; i <= 0xFF; i++) sum += freqs0[i]; if (sum != 0) { res = false; goto end; } // Check rules for first 2 bytes for (int i = 0; i < 256; i++) { // Exclude < 0xE0A0 || > 0xE0BF if ((i < 0xA0) || (i > 0xBF)) sum += freqs1[0xE0 * 256 + i]; // Exclude < 0xED80 || > 0xEDE9F if ((i < 0x80) || (i > 0x9F)) sum += freqs1[0xED * 256 + i]; // Exclude < 0xF090 || > 0xF0BF if ((i < 0x90) || (i > 0xBF)) sum += freqs1[0xF0 * 256 + i]; // Exclude < 0xF480 || > 0xF48F if ((i < 0x80) || (i > 0x8F)) sum += freqs1[0xF4 * 256 + i]; if ((i < 0x80) || (i > 0xBF)) { // Exclude < 0x??80 || > 0x??BF with ?? in [C2..DF] for (int j = 0xC2; j <= 0xDF; j++) sum += freqs1[j * 256 + i]; // Exclude < 0x??80 || > 0x??BF with ?? in [E1..EC] for (int j = 0xE1; j <= 0xEC; j++) sum += freqs1[j * 256 + i]; // Exclude < 0x??80 || > 0x??BF with ?? in [F1..F3] sum += freqs1[0xF1 * 256 + i]; sum += freqs1[0xF2 * 256 + i]; sum += freqs1[0xF3 * 256 + i]; // Exclude < 0xEE80 || > 0xEEBF sum += freqs1[0xEE * 256 + i]; // Exclude < 0xEF80 || > 0xEFBF sum += freqs1[0xEF * 256 + i]; } else { // Count non-primary bytes sum2 += freqs0[i]; } if (sum != 0) { res = false; break; } } end: // Ad-hoc threshold res &= (sum2 >= uint(count / 8)); return res == true ? TextCodec::MASK_NOT_TEXT | byte(Global::UTF8) : TextCodec::MASK_NOT_TEXT; } TextCodec::TextCodec() { _delegate = nullptr; _pCtx = nullptr; _encodingType = 0; _bsVersion = 7; setEncodingType(1); } TextCodec::TextCodec(Context& ctx) { _delegate = nullptr; _pCtx = &ctx; _encodingType = 0; _bsVersion = ctx.getInt("bsVersion", 7); setEncodingType(ctx.getInt("textcodec", 1)); } void TextCodec::setEncodingType(int encodingType) { if ((_delegate != nullptr) && (_encodingType == encodingType)) return; delete _delegate; _encodingType = encodingType; _delegate = (encodingType == 1) ? static_cast*>(_pCtx == nullptr ? static_cast*>(new TextCodec1()) : static_cast*>(new TextCodec1(*_pCtx))) : static_cast*>(_pCtx == nullptr ? static_cast*>(new TextCodec2()) : static_cast*>(new TextCodec2(*_pCtx))); } bool TextCodec::forward(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if ((count < MIN_BLOCK_SIZE) || (count > MAX_BLOCK_SIZE)) return false; if (!SliceArray::isValid(input)) throw invalid_argument("TextCodec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("TextCodec: Invalid output block"); if (input._array == output._array) return false; const int dstIdx = output._index; const bool res = _delegate->forward(input, output, count); if ((res == true) && (_bsVersion >= 7)) { if (_encodingType == 1) output._array[dstIdx] &= ~TextCodec::MASK_TEXT_CODEC; else output._array[dstIdx] |= TextCodec::MASK_TEXT_CODEC; } return res; } bool TextCodec::inverse(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (count > MAX_BLOCK_SIZE) // ! no min return false; if (!SliceArray::isValid(input)) throw invalid_argument("TextCodec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("TextCodec: Invalid output block"); if (input._array == output._array) return false; if ((count < 2) || (input._index + count > input._length)) return false; // Since bsVersion 7, the transform header stores which TextCodec variant // produced the bitstream, so switch delegate before decoding payload bytes. if (_bsVersion >= 7) { const int encodingType = ((input._array[input._index] & TextCodec::MASK_TEXT_CODEC) == byte(0)) ? 1 : 2; setEncodingType(encodingType); } return _delegate->inverse(input, output, count); } TextCodec1::TextCodec1() { _logHashSize = TextCodec::LOG_HASHES_SIZE; _dictSize = 1 << 13; _dictMap = nullptr; _dictList = nullptr; _hashMask = (1 << _logHashSize) - 1; _staticDictSize = TextCodec::STATIC_DICT_WORDS; _isCRLF = false; _escapes[0] = TextCodec::ESCAPE_TOKEN2; _escapes[1] = TextCodec::ESCAPE_TOKEN1; _pCtx = nullptr; } TextCodec1::TextCodec1(Context& ctx) { // Actual block size const int blockSize = ctx.getInt("blockSize", 0); const int log = blockSize >= 8 ? max(min(Global::log2(uint32(blockSize / 8)), 26), 13) : 13; _logHashSize = ctx.getString("entropy") == "TPAQX" ? log + 1 : log; _dictSize = 1 << 13; _dictMap = nullptr; _dictList = nullptr; _hashMask = (1 << _logHashSize) - 1; _staticDictSize = TextCodec::STATIC_DICT_WORDS; _isCRLF = false; _escapes[0] = TextCodec::ESCAPE_TOKEN2; _escapes[1] = TextCodec::ESCAPE_TOKEN1; _pCtx = &ctx; } void TextCodec1::reset(int count) { // Select an appropriate initial dictionary size const int allocatedSize = _dictSize; const int log = count < 1024 ? 13 : max(min(Global::log2(uint32(count / 128)), 18), 13); _dictSize = max(TextCodec::STATIC_DICT_WORDS + 2, 1 << log); const int mapSize = 1 << _logHashSize; if (_dictMap == nullptr) _dictMap = new DictEntry*[mapSize]; for (int i = 0; i < mapSize; i++) _dictMap[i] = nullptr; if ((_dictList == nullptr) || (allocatedSize < _dictSize)) { DictEntry* newDict = new DictEntry[_dictSize]; delete[] _dictList; _dictList = newDict; #if __cplusplus >= 201103L memcpy(&_dictList[0], &TextCodec::STATIC_DICTIONARY[0], sizeof(TextCodec::STATIC_DICTIONARY)); #else for (int i = 0; i < TextCodec::STATIC_DICT_WORDS; i++) _dictList[i] = TextCodec::STATIC_DICTIONARY[i]; #endif // Add special entries at end of static dictionary _staticDictSize = TextCodec::STATIC_DICT_WORDS; _dictList[_staticDictSize] = DictEntry(&_escapes[0], 0, _staticDictSize, 1); _dictList[_staticDictSize + 1] = DictEntry(&_escapes[1], 0, _staticDictSize + 1, 1); _staticDictSize += 2; } for (int i = 0; i < _staticDictSize; i++) _dictMap[_dictList[i]._hash & _hashMask] = &_dictList[i]; // Pre-allocate all dictionary entries for (int i = _staticDictSize; i < _dictSize; i++) _dictList[i] = DictEntry(nullptr, 0, i); } bool TextCodec1::forward(SliceArray& input, SliceArray& output, int count) { if (output._length - output._index < getMaxEncodedLength(count)) return false; const byte* src = &input._array[input._index]; byte* dst = &output._array[output._index]; int srcIdx = 0; int dstIdx = 0; if (_pCtx != nullptr) { Global::DataType dt = (Global::DataType) _pCtx->getInt("dataType", Global::UNDEFINED); // Filter out most types. Still check binaries which may contain significant parts of text if ((dt != Global::UNDEFINED) && (dt != Global::TEXT) && (dt != Global::BIN)) return false; } uint freqs[256] = { 0 }; byte mode = TextCodec::computeStats(&src[srcIdx], count, freqs, true); // Not text ? if ((mode & TextCodec::MASK_NOT_TEXT) != byte(0)) { if (_pCtx != nullptr) _pCtx->putInt("dataType", Global::DataType(mode & TextCodec::MASK_DT)); return false; } if (_pCtx != nullptr) _pCtx->putInt("dataType", Global::TEXT); reset(count); const int srcEnd = count; const int dstEnd = getMaxEncodedLength(count); const int dstEnd4 = dstEnd - 4; int emitAnchor = 0; // never less than 0 int words = _staticDictSize; // DOS encoded end of line (CR+LF) ? _isCRLF = int(mode & TextCodec::MASK_CRLF) != 0; dst[dstIdx++] = mode; bool res = true; while ((srcIdx < srcEnd) && (src[srcIdx] == TextCodec::SP)) { dst[dstIdx++] = TextCodec::SP; srcIdx++; emitAnchor++; } int delimAnchor = TextCodec::isText(src[srcIdx]) ? srcIdx - 1 : srcIdx; // previous delimiter uint h1 = TextCodec::HASH1; uint h2 = TextCodec::HASH1; while (srcIdx < srcEnd) { const byte cur = src[srcIdx]; const int8 cType = TextCodec::getType(cur); if (cType == 0) { if (srcIdx - delimAnchor == 1) { h1 = uint(TextCodec::HASH1) * uint(TextCodec::HASH1) ^ uint(cur) * uint(TextCodec::HASH2); h2 = uint(TextCodec::HASH1) * uint(TextCodec::HASH1) ^ (uint(cur) ^ 0x20U) * uint(TextCodec::HASH2); } else { h1 = h1 * TextCodec::HASH1 ^ uint(cur) * TextCodec::HASH2; h2 = h2 * TextCodec::HASH1 ^ uint(cur) * TextCodec::HASH2; } srcIdx++; continue; } if ((srcIdx > delimAnchor + 2) && (cType > 0)) { // At least 2 letters const int length = srcIdx - delimAnchor - 1; if (length <= TextCodec::MAX_WORD_LENGTH) { // Check word in dictionary prefetchRead(&_dictMap[h1 & _hashMask]); DictEntry* pe = nullptr; DictEntry* pe1 = _dictMap[h1 & _hashMask]; if ((pe1 != nullptr) && (pe1->_hash == h1) && ((pe1->_data >> 24) == length)) pe = pe1; else { prefetchRead(&_dictMap[h2 & _hashMask]); DictEntry* pe2 = _dictMap[h2 & _hashMask]; if ((pe2 != nullptr) && (pe2->_hash == h2) && ((pe2->_data >> 24) == length)) pe = pe2; } // Check for hash collisions if ((pe != nullptr) && (KANZI_UNLIKELY(!TextCodec::sameWords(&pe->_ptr[1], &src[delimAnchor + 2], length - 1)))) pe = nullptr; if (pe == nullptr) { // Word not found in the dictionary or hash collision. // Replace entry if not in static dictionary if (((length > 3) || ((length == 3) && (words < TextCodec::THRESHOLD2))) && (pe1 == nullptr)) { DictEntry* pe3 = &_dictList[words]; if ((pe3->_data & TextCodec::MASK_LENGTH) >= _staticDictSize) { // Reuse old entry _dictMap[pe3->_hash & _hashMask] = nullptr; pe3->_ptr = &src[delimAnchor + 1]; pe3->_hash = h1; pe3->_data = (length << 24) | words; } // Update hash map _dictMap[h1 & _hashMask] = pe3; words++; // Dictionary full ? Expand or reset index to end of static dictionary if (words >= _dictSize) { if (expandDictionary() == false) words = _staticDictSize; } } } else { // Word found in the dictionary // Skip space if only delimiter between 2 word references if ((emitAnchor != delimAnchor) || (src[delimAnchor] != byte(' '))) { const int dIdx = emitSymbols(&src[emitAnchor], &dst[dstIdx], delimAnchor + 1 - emitAnchor, dstEnd - dstIdx); if (dIdx < 0) { res = false; break; } dstIdx += dIdx; } if (dstIdx >= dstEnd4) { res = false; break; } dst[dstIdx++] = (pe == pe1) ? TextCodec::ESCAPE_TOKEN1 : TextCodec::ESCAPE_TOKEN2; dstIdx += emitWordIndex(&dst[dstIdx], pe->_data & TextCodec::MASK_LENGTH); emitAnchor = delimAnchor + 1 + int(pe->_data >> 24); } } } // Reset delimiter position delimAnchor = srcIdx; srcIdx++; } if (res == true) { // Emit last symbols const int dIdx = emitSymbols(&src[emitAnchor], &dst[dstIdx], srcEnd - emitAnchor, dstEnd - dstIdx); if (dIdx < 0) res = false; else dstIdx += dIdx; res &= (srcIdx == srcEnd); } output._index += dstIdx; input._index += srcIdx; return res; } bool TextCodec1::expandDictionary() { if (_dictSize >= TextCodec::MAX_DICT_SIZE) return false; DictEntry* newDict = new DictEntry[_dictSize * 2]; memcpy(static_cast(&newDict[0]), &_dictList[0], sizeof(DictEntry) * _dictSize); for (int i = _dictSize; i < _dictSize * 2; i++) newDict[i] = DictEntry(nullptr, 0, i); delete[] _dictList; _dictList = newDict; // Reset map (values must point to addresses of new DictEntry items) for (int i = 0; i < _dictSize; i++) { _dictMap[_dictList[i]._hash & _hashMask] = &_dictList[i]; } _dictSize <<= 1; return true; } int TextCodec1::emitSymbols(const byte src[], byte dst[], const int srcEnd, const int dstEnd) const { int dstIdx = 0; for (int i = 0; i < srcEnd; i++) { if (dstIdx >= dstEnd) return -1; // Work around incorrect warning by GCC 7.x.x with C++17 #ifdef __GNUC__ #if (__GNUC__ == 7) && (__cplusplus > 201402L) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wswitch" #endif #endif const byte cur = src[i]; switch (cur) { case TextCodec::ESCAPE_TOKEN1: case TextCodec::ESCAPE_TOKEN2: { // Emit special word dst[dstIdx++] = TextCodec::ESCAPE_TOKEN1; const int idx = (cur == TextCodec::ESCAPE_TOKEN1) ? _staticDictSize - 1 : _staticDictSize - 2; int lenIdx = 1; if (idx >= TextCodec::THRESHOLD1) lenIdx = (idx >= TextCodec::THRESHOLD2) ? 3 : 2; if (dstIdx + lenIdx >= dstEnd) return -1; dstIdx += emitWordIndex(&dst[dstIdx], idx); break; } case TextCodec::CR: if (_isCRLF == false) dst[dstIdx++] = cur; break; default: dst[dstIdx++] = cur; } } // Work around incorrect warning by GCC 7.x.x with C++17 #ifdef __GNUC__ #if (__GNUC__ == 7) && (__cplusplus > 201402L) #pragma GCC diagnostic pop #endif #endif return dstIdx; } int TextCodec1::emitWordIndex(byte dst[], int val) { // Emit word index (varint 5 bits + 7 bits + 7 bits) if (val >= TextCodec::THRESHOLD1) { if (val >= TextCodec::THRESHOLD2) { dst[0] = byte(0xE0 | (val >> 14)); dst[1] = byte(0x80 | (val >> 7)); dst[2] = byte(0x7F & val); return 3; } dst[0] = byte(0x80 | (val >> 7)); dst[1] = byte(0x7F & val); return 2; } dst[0] = byte(val); return 1; } bool TextCodec1::inverse(SliceArray& input, SliceArray& output, int count) { if ((count < 2) || (input._index + count > input._length)) return false; reset(output._length); const byte* src = &input._array[input._index]; byte* dst = &output._array[output._index]; _isCRLF = int(src[0] & TextCodec::MASK_CRLF) != 0; const bool isCRLF = _isCRLF; int srcIdx = 1; int dstIdx = 0; const int srcEnd = count; const int dstEnd = output._length - output._index; int delimAnchor = TextCodec::isText(src[srcIdx]) ? srcIdx - 1 : srcIdx; // previous delimiter int words = _staticDictSize; bool wordRun = false; bool res = true; while ((srcIdx < srcEnd) && (dstIdx < dstEnd)) { const byte cur = src[srcIdx]; const int8 cType = TextCodec::getType(cur); if (cType == 0) { dst[dstIdx] = src[srcIdx]; srcIdx++; dstIdx++; continue; } if ((srcIdx > delimAnchor + 3) && (cType > 0)) { const int length = srcIdx - delimAnchor - 1; // length > 2 if (length <= TextCodec::MAX_WORD_LENGTH) { uint h1 = TextCodec::HASH1; h1 = h1 * TextCodec::HASH1 ^ uint(src[delimAnchor + 1]) * TextCodec::HASH2; h1 = h1 * TextCodec::HASH1 ^ uint(src[delimAnchor + 2]) * TextCodec::HASH2; for (int i = delimAnchor + 3; i < srcIdx; i++) h1 = h1 * TextCodec::HASH1 ^ uint(src[i]) * TextCodec::HASH2; // Lookup word in dictionary DictEntry* pe = nullptr; DictEntry* pe1 = _dictMap[h1 & _hashMask]; // A hash collision check is not needed here: insertion is only allowed when // the hash slot is null, so the candidate contents cannot affect the outcome. if ((pe1 != nullptr) && (pe1->_hash == h1) && ((pe1->_data >> 24) == length)) pe = pe1; if (pe == nullptr) { // Word not found in the dictionary or hash collision. // Replace entry if not in static dictionary if (((length > 3) || (words < TextCodec::THRESHOLD2)) && (pe1 == nullptr)) { DictEntry& e = _dictList[words]; if ((e._data & TextCodec::MASK_LENGTH) >= _staticDictSize) { // Reuse old entry _dictMap[e._hash & _hashMask] = nullptr; e._ptr = &src[delimAnchor + 1]; e._hash = h1; e._data = (length << 24) | words; } _dictMap[h1 & _hashMask] = &e; words++; // Dictionary full ? Expand or reset index to end of static dictionary if (words >= _dictSize) { if (expandDictionary() == false) words = _staticDictSize; } } } } } srcIdx++; if ((cur == TextCodec::ESCAPE_TOKEN1) || (cur == TextCodec::ESCAPE_TOKEN2)) { // Word in dictionary // Read word index (varint 5 bits + 7 bits + 7 bits) if (srcIdx >= srcEnd) { res = false; break; } int idx = int(src[srcIdx++]); if (idx >= 128) { if (srcIdx >= srcEnd) { res = false; break; } const int idx2 = int(src[srcIdx++]); if (idx2 >= 128) { if (srcIdx >= srcEnd) { res = false; break; } idx = ((idx & 0x1F) << 14) | ((idx2 & 0x7F) << 7) | int(src[srcIdx]); srcIdx++; } else { idx = ((idx & 0x7F) << 7) | idx2; } if (KANZI_UNLIKELY(idx >= _dictSize)) { res = false; break; } } const int length = (_dictList[idx]._data >> 24) & 0xFF; // Emit word if (length > 1) { // Add space if only delimiter between 2 words (not an escaped delimiter) if (wordRun == true) dst[dstIdx++] = TextCodec::SP; // Regular word entry wordRun = true; delimAnchor = srcIdx; } else { if (KANZI_UNLIKELY(length == 0)) { res = false; break; } // Escape entry wordRun = false; delimAnchor = srcIdx - 1; } // Sanity check if (KANZI_UNLIKELY(dstIdx + length > dstEnd)) { res = false; break; } memcpy(&dst[dstIdx], _dictList[idx]._ptr, length); // Flip case of first character ? if (cur == TextCodec::ESCAPE_TOKEN2) dst[dstIdx] ^= byte(0x20); dstIdx += length; } else { wordRun = false; delimAnchor = srcIdx - 1; if ((isCRLF == true) && (cur == TextCodec::LF)) { dst[dstIdx++] = TextCodec::CR; if (dstIdx >= dstEnd) { res = false; break; } } dst[dstIdx++] = cur; } } output._index += dstIdx; input._index += srcIdx; return (res == true) && (srcIdx == srcEnd); } TextCodec2::TextCodec2() { _logHashSize = TextCodec::LOG_HASHES_SIZE; _dictSize = 1 << 13; _dictMap = nullptr; _dictList = nullptr; _hashMask = (1 << _logHashSize) - 1; _staticDictSize = TextCodec::STATIC_DICT_WORDS; _isCRLF = false; _pCtx = nullptr; _bsVersion = 6; } TextCodec2::TextCodec2(Context& ctx) { const int blockSize = ctx.getInt("blockSize", 0); const int log = blockSize >= 32 ? max(min(Global::log2(uint32(blockSize / 32)), 24), 13) : 13; _logHashSize = ctx.getString("entropy") == "TPAQX" ? log + 1 : log; _dictSize = 1 << 13; _dictMap = nullptr; _dictList = nullptr; _hashMask = (1 << _logHashSize) - 1; _staticDictSize = TextCodec::STATIC_DICT_WORDS; _isCRLF = false; _pCtx = &ctx; _bsVersion = ctx.getInt("bsVersion"); } void TextCodec2::reset(int count) { // Select an appropriate initial dictionary size const int allocatedSize = _dictSize; const int log = count < 1024 ? 13 : max(min(Global::log2(uint32(count / 128)), 18), 13); _dictSize = max(TextCodec::STATIC_DICT_WORDS, 1 << log); const int mapSize = 1 << _logHashSize; if (_dictMap == nullptr) _dictMap = new DictEntry*[mapSize]; for (int i = 0; i < mapSize; i++) _dictMap[i] = nullptr; if ((_dictList == nullptr) || (allocatedSize < _dictSize)) { DictEntry* newDict = new DictEntry[_dictSize]; delete[] _dictList; _dictList = newDict; #if __cplusplus >= 201103L memcpy(&_dictList[0], &TextCodec::STATIC_DICTIONARY[0], sizeof(TextCodec::STATIC_DICTIONARY)); #else for (int i = 0; i < TextCodec::STATIC_DICT_WORDS; i++) _dictList[i] = TextCodec::STATIC_DICTIONARY[i]; #endif } for (int i = 0; i < _staticDictSize; i++) _dictMap[_dictList[i]._hash & _hashMask] = &_dictList[i]; // Pre-allocate all dictionary entries for (int i = _staticDictSize; i < _dictSize; i++) _dictList[i] = DictEntry(nullptr, 0, i); } bool TextCodec2::forward(SliceArray& input, SliceArray& output, int count) { if (output._length - output._index < getMaxEncodedLength(count)) return false; const byte* src = &input._array[input._index]; byte* dst = &output._array[output._index]; if (_pCtx != nullptr) { Global::DataType dt = (Global::DataType) _pCtx->getInt("dataType", Global::UNDEFINED); // Filter out most types. Still check binaries which may contain significant parts of text if ((dt != Global::UNDEFINED) && (dt != Global::TEXT) && (dt != Global::BIN)) return false; } uint freqs[256] = { 0 }; byte mode = TextCodec::computeStats(&src[0], count, freqs, false); // Not text ? if ((mode & TextCodec::MASK_NOT_TEXT) != byte(0)) { if (_pCtx != nullptr) _pCtx->putInt("dataType", Global::DataType(mode & TextCodec::MASK_DT)); return false; } if (_pCtx != nullptr) _pCtx->putInt("dataType", Global::TEXT); reset(count); const int srcEnd = count; const int dstEnd = getMaxEncodedLength(count); const int dstEnd3 = dstEnd - 3; int emitAnchor = 0; int words = _staticDictSize; // DOS encoded end of line (CR+LF) ? _isCRLF = (mode & TextCodec::MASK_CRLF) != byte(0); dst[0] = mode; bool res = true; int srcIdx = 0; int dstIdx = 1; while ((srcIdx < srcEnd) && (src[srcIdx] == TextCodec::SP)) { dst[dstIdx++] = TextCodec::SP; srcIdx++; emitAnchor++; } int delimAnchor = TextCodec::isText(src[srcIdx]) ? srcIdx - 1 : srcIdx; // previous delimiter uint h1 = TextCodec::HASH1; uint h2 = TextCodec::HASH1; while (srcIdx < srcEnd) { const byte cur = src[srcIdx]; const int8 cType = TextCodec::getType(cur); if (cType == 0) { if (srcIdx - delimAnchor == 1) { h1 = uint(TextCodec::HASH1) * uint(TextCodec::HASH1) ^ uint(cur) * uint(TextCodec::HASH2); h2 = uint(TextCodec::HASH1) * uint(TextCodec::HASH1) ^ (uint(cur) ^ 0x20U) * uint(TextCodec::HASH2); } else { h1 = h1 * TextCodec::HASH1 ^ uint(cur) * TextCodec::HASH2; h2 = h2 * TextCodec::HASH1 ^ uint(cur) * TextCodec::HASH2; } srcIdx++; continue; } if ((srcIdx > delimAnchor + 2) && (cType > 0)) { const int length = srcIdx - delimAnchor - 1; if (length <= TextCodec::MAX_WORD_LENGTH) { // Check word in dictionary prefetchRead(&_dictMap[h1 & _hashMask]); DictEntry* pe = nullptr; DictEntry* pe1 = _dictMap[h1 & _hashMask]; if ((pe1 != nullptr) && (pe1->_hash == h1) && ((pe1->_data >> 24) == length)) pe = pe1; else { prefetchRead(&_dictMap[h2 & _hashMask]); DictEntry* pe2 = _dictMap[h2 & _hashMask]; if ((pe2 != nullptr) && (pe2->_hash == h2) && ((pe2->_data >> 24) == length)) pe = pe2; } // Check for hash collisions if ((pe != nullptr) && (KANZI_UNLIKELY(!TextCodec::sameWords(&pe->_ptr[1], &src[delimAnchor + 2], length - 1)))) pe = nullptr; if (pe == nullptr) { // Word not found in the dictionary or hash collision. // Replace entry if not in static dictionary if (((length > 3) || ((length == 3) && (words < TextCodec::THRESHOLD2))) && (pe1 == nullptr)) { DictEntry* pe3 = &_dictList[words]; if ((pe3->_data & TextCodec::MASK_LENGTH) >= _staticDictSize) { // Reuse old entry _dictMap[pe3->_hash & _hashMask] = nullptr; pe3->_ptr = &src[delimAnchor + 1]; pe3->_hash = h1; pe3->_data = (length << 24) | words; } // Update hash map _dictMap[h1 & _hashMask] = pe3; words++; // Dictionary full ? Expand or reset index to end of static dictionary if (words >= _dictSize) { if (expandDictionary() == false) words = _staticDictSize; } } } else { // Word found in the dictionary // Skip space if only delimiter between 2 word references if ((emitAnchor != delimAnchor) || (src[delimAnchor] != TextCodec::SP)) { const int dIdx = emitSymbols(&src[emitAnchor], &dst[dstIdx], delimAnchor + 1 - emitAnchor, dstEnd - dstIdx); if (dIdx < 0) { res = false; break; } dstIdx += dIdx; } if (dstIdx >= dstEnd3) { res = false; break; } // Case flip is encoded as 0x80 dst[dstIdx] = TextCodec::MASK_FLIP_CASE; dstIdx += (pe == pe1 ? 0 : 1); dstIdx += emitWordIndex(&dst[dstIdx], pe->_data & TextCodec::MASK_LENGTH); emitAnchor = delimAnchor + 1 + (pe->_data >> 24); } } } // Reset delimiter position delimAnchor = srcIdx; srcIdx++; } if (res == true) { // Emit last symbols const int dIdx = emitSymbols(&src[emitAnchor], &dst[dstIdx], srcEnd - emitAnchor, dstEnd - dstIdx); if (dIdx < 0) res = false; else dstIdx += dIdx; res &= (srcIdx == srcEnd); } output._index += dstIdx; input._index += srcIdx; return res; } bool TextCodec2::expandDictionary() { if (_dictSize >= TextCodec::MAX_DICT_SIZE) return false; DictEntry* newDict = new DictEntry[_dictSize * 2]; memcpy(static_cast(&newDict[0]), &_dictList[0], sizeof(DictEntry) * _dictSize); for (int i = _dictSize; i < _dictSize * 2; i++) newDict[i] = DictEntry(nullptr, 0, i); delete[] _dictList; _dictList = newDict; // Reset map (values must point to addresses of new DictEntry items) for (int i = 0; i < _dictSize; i++) { _dictMap[_dictList[i]._hash & _hashMask] = &_dictList[i]; } _dictSize <<= 1; return true; } int TextCodec2::emitSymbols(const byte src[], byte dst[], const int srcEnd, const int dstEnd) const { // Work around incorrect warning by GCC 7.x.x with C++17 #ifdef __GNUC__ #if (__GNUC__ == 7) && (__cplusplus > 201402L) #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wswitch" #endif #endif int dstIdx = 0; if (2 * srcEnd < dstEnd) { for (int i = 0; i < srcEnd; i++) { switch (src[i]) { case TextCodec::ESCAPE_TOKEN1: dst[dstIdx++] = TextCodec::ESCAPE_TOKEN1; dst[dstIdx++] = TextCodec::ESCAPE_TOKEN1; break; case TextCodec::CR: if (_isCRLF == false) dst[dstIdx++] = src[i]; break; default: dst[dstIdx] = TextCodec::ESCAPE_TOKEN1; dstIdx += int(src[i] >> 7); dst[dstIdx++] = src[i]; } } } else { for (int i = 0; i < srcEnd; i++) { switch (src[i]) { case TextCodec::ESCAPE_TOKEN1: if (dstIdx >= dstEnd - 1) return -1; dst[dstIdx++] = TextCodec::ESCAPE_TOKEN1; dst[dstIdx++] = TextCodec::ESCAPE_TOKEN1; break; case TextCodec::CR: if (_isCRLF == false) { if (dstIdx >= dstEnd) return -1; dst[dstIdx++] = src[i]; } break; default: if (src[i] >= kanzi::byte(128)) { if (dstIdx >= dstEnd) return -1; dst[dstIdx++] = TextCodec::ESCAPE_TOKEN1; } if (dstIdx >= dstEnd) return -1; dst[dstIdx++] = src[i]; } } } // Work around incorrect warning by GCC 7.x.x with C++17 #ifdef __GNUC__ #if (__GNUC__ == 7) && (__cplusplus > 201402L) #pragma GCC diagnostic pop #endif #endif return dstIdx; } int TextCodec2::emitWordIndex(kanzi::byte dst[], int wIdx) { // 0x80 is reserved to first symbol case flip if (wIdx < V7_INDEX_BASE2) { dst[0] = kanzi::byte(0x81 + wIdx); return 1; } if (wIdx < V7_INDEX_BASE3) { // Encode the rank relative to the one-byte range. wIdx -= V7_INDEX_BASE2; dst[0] = kanzi::byte(0xC0 | (wIdx >> 8)); dst[1] = kanzi::byte(wIdx); return 2; } // Encode the rank relative to the one- and two-byte ranges. wIdx -= V7_INDEX_BASE3; dst[0] = kanzi::byte(0xF0 | (wIdx >> 16)); dst[1] = kanzi::byte(wIdx >> 8); dst[2] = kanzi::byte(wIdx); return 3; } bool TextCodec2::inverse(SliceArray& input, SliceArray& output, int count) { if ((count < 2) || (input._index + count > input._length)) return false; reset(output._length); const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; _isCRLF = (src[0] & TextCodec::MASK_CRLF) != kanzi::byte(0); const bool isCRLF = _isCRLF; int srcIdx = 1; int dstIdx = 0; const int srcEnd = count; const int dstEnd = output._length - output._index; int delimAnchor = TextCodec::isText(src[srcIdx]) ? srcIdx - 1 : srcIdx; // previous delimiter int words = _staticDictSize; bool wordRun = false; bool res = true; const bool oldEncoding = _bsVersion < 6; while ((srcIdx < srcEnd) && (dstIdx < dstEnd)) { kanzi::byte cur = src[srcIdx]; const int8 cType = TextCodec::getType(cur); if (cType == 0) { dst[dstIdx] = src[srcIdx]; srcIdx++; dstIdx++; continue; } if ((srcIdx > delimAnchor + 3) && (cType > 0)) { const int length = srcIdx - delimAnchor - 1; // length > 2 if (length <= TextCodec::MAX_WORD_LENGTH) { uint h1 = TextCodec::HASH1; h1 = h1 * TextCodec::HASH1 ^ uint(src[delimAnchor + 1]) * TextCodec::HASH2; h1 = h1 * TextCodec::HASH1 ^ uint(src[delimAnchor + 2]) * TextCodec::HASH2; for (int i = delimAnchor + 3; i < srcIdx; i++) h1 = h1 * TextCodec::HASH1 ^ uint(src[i]) * TextCodec::HASH2; // Lookup word in dictionary DictEntry* pe = nullptr; DictEntry* pe1 = _dictMap[h1 & _hashMask]; // A hash collision check is not needed here: insertion is only allowed when // the hash slot is null, so the candidate contents cannot affect the outcome. if ((pe1 != nullptr) && (pe1->_hash == h1) && ((pe1->_data >> 24) == length)) pe = pe1; if (pe == nullptr) { // Word not found in the dictionary or hash collision. // Replace entry if not in static dictionary if (((length > 3) || (words < TextCodec::THRESHOLD2)) && (pe1 == nullptr)) { DictEntry& e = _dictList[words]; if ((e._data & TextCodec::MASK_LENGTH) >= _staticDictSize) { // Reuse old entry _dictMap[e._hash & _hashMask] = nullptr; e._ptr = &src[delimAnchor + 1]; e._hash = h1; e._data = (length << 24) | words; } _dictMap[h1 & _hashMask] = &e; words++; // Dictionary full ? Expand or reset index to end of static dictionary if (words >= _dictSize) { if (expandDictionary() == false) words = _staticDictSize; } } } } } srcIdx++; kanzi::byte flipMask = kanzi::byte(0); if (cur >= TextCodec::MASK_80) { // Word in dictionary int idx; if (oldEncoding == true) { // Read word index (varint 5 bits + 7 bits + 7 bits) flipMask = cur & TextCodec::MASK_20; idx = int(cur & TextCodec::MASK_1F); if ((cur & TextCodec::MASK_40) != kanzi::byte(0)) { if (KANZI_UNLIKELY(srcIdx >= srcEnd)) { res = false; break; } const int idx2 = int(src[srcIdx++]); if (idx2 >= 128) { if (KANZI_UNLIKELY(srcIdx >= srcEnd)) { res = false; break; } idx = (idx << 14) | ((idx2 & 0x7F) << 7) | int(src[srcIdx]); srcIdx++; } else { idx = (idx << 7) | idx2; } // Sanity check if (KANZI_UNLIKELY(idx >= _dictSize)) { res = false; break; } } } else { const bool rankedEncoding = _bsVersion >= 7; if (cur == TextCodec::MASK_80) { // Flip first char case flipMask = TextCodec::MASK_20; if (KANZI_UNLIKELY(srcIdx >= srcEnd)) { res = false; break; } cur = src[srcIdx++]; } // Read word index // 10xxxxxx => 1 byte // 110xxxxx => 2 bytes // 1111xxxx => 3 bytes idx = int(cur) & 0x7F; const bool oneByte = idx < 64; if (idx >= 64) { const bool threeBytes = idx >= 112; if (threeBytes) { if (KANZI_UNLIKELY(srcEnd - srcIdx < 2)) { res = false; break; } idx = ((idx & 0x0F) << 16) | (int(src[srcIdx]) << 8) | int(src[srcIdx + 1]); srcIdx += 2; } else { if (KANZI_UNLIKELY(srcIdx >= srcEnd)) { res = false; break; } idx = ((idx & 0x1F) << 8) | int(src[srcIdx]); srcIdx++; } if (rankedEncoding == true) { // Add the rank of the shorter forms. This prevents // zero or negative dictionary indexes for malformed // multi-byte encodings such as C0 00 or F0 00 00. idx += threeBytes ? V7_INDEX_BASE3 : V7_INDEX_BASE2; if (KANZI_UNLIKELY(idx >= _dictSize)) { res = false; break; } } else if (KANZI_UNLIKELY(idx > _dictSize)) { // V6 uses the original one-based dictionary index. res = false; break; } } if (KANZI_UNLIKELY(idx == 0)) { res = false; break; } idx -= rankedEncoding ? int(oneByte) : 1; } const int length = (_dictList[idx]._data >> 24) & 0xFF; // Emit word if (length > 1) { // Add space if only delimiter between 2 words (not an escaped delimiter) if (wordRun == true) dst[dstIdx++] = TextCodec::SP; // Regular word entry wordRun = true; delimAnchor = srcIdx; } else { if (KANZI_UNLIKELY(length == 0)) { res = false; break; } // Escape entry wordRun = false; delimAnchor = srcIdx - 1; } // Sanity check if (KANZI_UNLIKELY(dstIdx + length > dstEnd)) { res = false; break; } memcpy(&dst[dstIdx], _dictList[idx]._ptr, length); // Flip case of first character ? dst[dstIdx] ^= flipMask; dstIdx += length; } else { if (cur == TextCodec::ESCAPE_TOKEN1) { if (KANZI_UNLIKELY(srcIdx >= srcEnd)) { res = false; break; } dst[dstIdx++] = src[srcIdx++]; } else { if ((isCRLF == true) && (cur == TextCodec::LF)) { dst[dstIdx++] = TextCodec::CR; if (KANZI_UNLIKELY(dstIdx >= dstEnd)) { res = false; break; } } dst[dstIdx++] = cur; } wordRun = false; delimAnchor = srcIdx - 1; } } output._index += dstIdx; input._index += srcIdx; return (res == true) && (srcIdx == srcEnd); } kanzi-cpp-2.6.0/src/transform/TextCodec.hpp000066400000000000000000000173251525576310500206310ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_TextCodec #define knz_TextCodec #include "../Context.hpp" #include "../Memory.hpp" #include "../Transform.hpp" namespace kanzi { class DictEntry FINAL { public: const byte* _ptr; // text data uint _hash; // full word hash int _data; // packed word length (8 MSB) + index in dictionary (24 LSB) DictEntry(); DictEntry(const byte* ptr, int hash, int idx, int length); #if __cplusplus < 201103L DictEntry(const DictEntry& de); DictEntry& operator=(const DictEntry& de); ~DictEntry() {} #else DictEntry(const DictEntry& de) = delete; DictEntry& operator=(const DictEntry& de) = delete; DictEntry(DictEntry&& de) noexcept = default; DictEntry& operator=(DictEntry&& de) noexcept = default; ~DictEntry() noexcept = default; #endif }; // Encode word indexes using a token class TextCodec1 FINAL : public Transform { public: TextCodec1(); TextCodec1(Context&); ~TextCodec1() { if (_dictList != nullptr) delete[] _dictList; if (_dictMap != nullptr) delete[] _dictMap; } bool forward(SliceArray& src, SliceArray& dst, int length); bool inverse(SliceArray& src, SliceArray& dst, int length); // Limit to 1 x srcLength and let the caller deal with // a failure when the output is too small int getMaxEncodedLength(int srcLen) const { return srcLen; } private: DictEntry** _dictMap; DictEntry* _dictList; byte _escapes[2]; int _staticDictSize; int _dictSize; int _logHashSize; int _hashMask; bool _isCRLF; // EOL = CR + LF Context* _pCtx; bool expandDictionary(); void reset(int count); static int emitWordIndex(byte dst[], int val); int emitSymbols(const byte src[], byte dst[], const int srcEnd, const int dstEnd) const; }; // Encode word indexes using a mask (0x80) class TextCodec2 FINAL : public Transform { public: TextCodec2(); TextCodec2(Context&); ~TextCodec2() { if (_dictList != nullptr) delete[] _dictList; if (_dictMap != nullptr) delete[] _dictMap; } bool forward(SliceArray& src, SliceArray& dst, int length); bool inverse(SliceArray& src, SliceArray& dst, int length); // Limit to 1 x srcLength and let the caller deal with // a failure when the output is too small int getMaxEncodedLength(int srcLen) const { return srcLen; } private: DictEntry** _dictMap; DictEntry* _dictList; int _staticDictSize; int _dictSize; int _logHashSize; int _hashMask; int _bsVersion; bool _isCRLF; // EOL = CR + LF Context* _pCtx; bool expandDictionary(); void reset(int count); static int emitWordIndex(byte dst[], int val); int emitSymbols(const byte src[], byte dst[], const int srcEnd, const int dstEnd) const; }; // Simple one-pass text codec that replaces words with indexes. // Generates a dynamic dictionary. class TextCodec FINAL : public Transform { friend class TextCodec1; friend class TextCodec2; public: static const int MAX_DICT_SIZE; static const int MAX_WORD_LENGTH; static const int MIN_BLOCK_SIZE; static const int MAX_BLOCK_SIZE; static const byte ESCAPE_TOKEN1; static const byte ESCAPE_TOKEN2; static const byte MASK_1F; static const byte MASK_3F; static const byte MASK_20; static const byte MASK_40; static const byte MASK_80; static const byte MASK_FLIP_CASE; TextCodec(); TextCodec(Context& ctx); ~TextCodec() { delete _delegate; } bool forward(SliceArray& src, SliceArray& dst, int length); bool inverse(SliceArray& src, SliceArray& dst, int length); int getMaxEncodedLength(int srcLen) const { return _delegate->getMaxEncodedLength(srcLen); } static int8 getType(byte val) { return CHAR_TYPE[uint8(val)]; } static bool isText(byte val) { return getType(val) == 0; } static bool isLowerCase(byte val) { return (val >= byte('a')) && (val <= byte('z')); } static bool isUpperCase(byte val) { return (val >= byte('A')) && (val <= byte('Z')); } static bool isDelimiter(byte val) { return getType(val) > 0; } private: static const int HASH1; static const int HASH2; static const byte CR; static const byte LF; static const byte SP; static const int THRESHOLD1; static const int THRESHOLD2; static const int THRESHOLD3; static const int THRESHOLD4; static const int LOG_HASHES_SIZE; static const byte MASK_NOT_TEXT; static const byte MASK_CRLF; static const byte MASK_XML_HTML; static const byte MASK_TEXT_CODEC; static const byte MASK_DT; static const int MASK_LENGTH; static bool init(int8 cType[256]); static int8 CHAR_TYPE[256]; static const bool INIT; static uint computeWordHash(const byte src[], int length); static bool sameWords(const byte src[], const byte dst[], int length); static byte computeStats(const byte block[], int count, uint freqs[], bool strict); static byte detectType(const uint freqs0[], const uint freqs1[], int count); // Common English words. static char DICT_EN_1024[]; // Static dictionary of 1024 entries. static DictEntry STATIC_DICTIONARY[1024]; static int createDictionary(char words[], int dictSize, DictEntry dict[], int maxWords, int startWord); static const int STATIC_DICT_WORDS; void setEncodingType(int encodingType); Transform* _delegate; Context* _pCtx; int _encodingType; int _bsVersion; }; inline DictEntry::DictEntry() : _ptr(nullptr) , _hash(0) , _data(0) { } inline DictEntry::DictEntry(const byte* ptr, int hash, int idx, int length = 0) : _ptr(ptr) , _hash(hash) , _data((length << 24) | idx) { } #if __cplusplus < 201103L inline DictEntry::DictEntry(const DictEntry& de) { _ptr = de._ptr; _hash = de._hash; _data = de._data; } inline DictEntry& DictEntry::operator=(const DictEntry& de) { _ptr = de._ptr; _hash = de._hash; _data = de._data; return *this; } #endif inline uint TextCodec::computeWordHash(const byte src[], int length) { uint h = HASH1; for (int i = 0; i < length; i++) h = h * HASH1 ^ uint(src[i]) * HASH2; return h; } inline bool TextCodec::sameWords(const byte src[], const byte dst[], int length) { while (length >= 4) { length -= 4; if (!KANZI_MEM_EQ4(&src[length], &dst[length])) return false; } while (length > 0) { length--; if (dst[length] != src[length]) return false; } return true; } } #endif kanzi-cpp-2.6.0/src/transform/TransformFactory.hpp000066400000000000000000000244621525576310500222520ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_TransformFactory #define knz_TransformFactory #include #include #include #include "../types.hpp" #include "../Context.hpp" #include "../util/strings.hpp" #include "AliasCodec.hpp" #include "BWTBlockCodec.hpp" #include "BWTS.hpp" #include "EXECodec.hpp" #include "FSDCodec.hpp" #include "LZCodec.hpp" #include "NullTransform.hpp" #include "ROLZCodec.hpp" #include "RLT.hpp" #include "SBRT.hpp" #include "SRT.hpp" #include "TextCodec.hpp" #include "TransformSequence.hpp" #include "UTFCodec.hpp" #include "ZRLT.hpp" namespace kanzi { template class TransformFactory { public: // Up to 64 transforms can be declared (6 bit index) enum TransformType { NONE_TYPE = 0, // Copy BWT_TYPE = 1, // Burrows Wheeler BWTS_TYPE = 2, // Burrows Wheeler Scott LZ_TYPE = 3, // Lempel Ziv SNAPPY_TYPE = 4, // Snappy (obsolete) RLT_TYPE = 5, // Run Length ZRLT_TYPE = 6, // Zero Run Length MTFT_TYPE = 7, // Move To Front RANK_TYPE = 8, // Rank EXE_TYPE = 9, // EXE codec DICT_TYPE = 10, // Text codec ROLZ_TYPE = 11, // ROLZ codec ROLZX_TYPE = 12, // ROLZ Extra codec SRT_TYPE = 13, // Sorted Rank LZP_TYPE = 14, // Lempel Ziv Predict MM_TYPE = 15, // Multimedia (FSD) codec LZX_TYPE = 16, // Lempel Ziv Extra UTF_TYPE = 17, // UTF Codec PACK_TYPE = 18, // Alias Codec DNA_TYPE = 19, // DNA Alias Codec RESERVED3 = 20, // Reserved RESERVED4 = 21, // Reserved RESERVED5 = 22 // Reserved }; static uint64 getType(const char* tName); static uint64 getTypeToken(const char* tName); static std::string getName(uint64 functionType); static int getTransformCount(uint64 functionType); static TransformSequence* newTransform(Context& ctx, uint64 functionType); private: TransformFactory() {} ~TransformFactory() {} static const int ONE_SHIFT = 6; // bits per transform static const int MAX_SHIFT = (8 - 1) * ONE_SHIFT; // 8 transforms static const int MASK = (1 << ONE_SHIFT) - 1; static Transform* newToken(Context& ctx, uint64 functionType); static const char* getNameToken(uint64 functionType); }; // The returned type contains 8 transform values template uint64 TransformFactory::getType(const char* tName) { std::string name(tName); size_t pos = name.find('+'); if (pos == std::string::npos) return getTypeToken(name.c_str()) << MAX_SHIFT; size_t prv = 0; int n = 0; uint64 res = 0; int shift = MAX_SHIFT; name += '+'; while (pos != std::string::npos) { n++; if (n > 8) { std::stringstream ss; ss << "Only 8 transforms allowed: " << name; throw std::invalid_argument(ss.str()); } std::string token = name.substr(prv, pos - prv); uint64 typeTk = getTypeToken(token.c_str()); // Skip null transform if (typeTk != NONE_TYPE) { res |= (typeTk << shift); shift -= ONE_SHIFT; } prv = pos + 1; pos = name.find('+', prv); } return res; } template uint64 TransformFactory::getTypeToken(const char* tName) { std::string name(tName); transform(name.begin(), name.end(), name.begin(), safeToUpper); if (name == "TEXT") return DICT_TYPE; if (name == "BWT") return BWT_TYPE; if (name == "BWTS") return BWTS_TYPE; if (name == "ROLZ") return ROLZ_TYPE; if (name == "ROLZX") return ROLZX_TYPE; if (name == "MTFT") return MTFT_TYPE; if (name == "ZRLT") return ZRLT_TYPE; if (name == "RLT") return RLT_TYPE; if (name == "SRT") return SRT_TYPE; if (name == "RANK") return RANK_TYPE; if (name == "LZ") return LZ_TYPE; if (name == "LZX") return LZX_TYPE; if (name == "LZP") return LZP_TYPE; if (name == "EXE") return EXE_TYPE; if (name == "UTF") return UTF_TYPE; if (name == "PACK") return PACK_TYPE; if (name == "DNA") return DNA_TYPE; if (name == "MM") return MM_TYPE; if (name == "NONE") return NONE_TYPE; std::stringstream ss; ss << "Unknown transform type: '" << name << "'"; throw std::invalid_argument(ss.str()); } template int TransformFactory::getTransformCount(uint64 functionType) { int count = 1; for (int i = 1; i < 8; i++) { if (((functionType >> (MAX_SHIFT - ONE_SHIFT * i)) & MASK) == NONE_TYPE) break; count++; } return count; } template TransformSequence* TransformFactory::newTransform(Context& ctx, uint64 functionType) { Transform* transforms[8]; int nbtr = 0; for (int i = 0; i < 8; i++) { transforms[i] = nullptr; const uint64 t = (functionType >> (MAX_SHIFT - ONE_SHIFT * i)) & MASK; if ((t != NONE_TYPE) || (i == 0)) transforms[nbtr++] = newToken(ctx, t); } return new TransformSequence(transforms, true); } template Transform* TransformFactory::newToken(Context& ctx, uint64 functionType) { switch (functionType) { case DICT_TYPE: { int textCodecType = 1; if (ctx.has("entropy")) { std::string entropyType = ctx.getString("entropy"); transform(entropyType.begin(), entropyType.end(), entropyType.begin(), safeToUpper); // Select text encoding based on entropy codec. if ((entropyType == "NONE") || (entropyType == "ANS0") || (entropyType == "HUFFMAN") || (entropyType == "RANGE")) textCodecType = 2; } ctx.putInt("textcodec", textCodecType); return new TextCodec(ctx); } case ROLZ_TYPE: return new ROLZCodec(ctx); case ROLZX_TYPE: return new ROLZCodec(ctx); case BWT_TYPE: return new BWTBlockCodec(ctx); case BWTS_TYPE: return new BWTS(ctx); case LZX_TYPE: ctx.putInt("lz", LZX_TYPE); return new LZCodec(ctx); case LZ_TYPE: ctx.putInt("lz", LZ_TYPE); return new LZCodec(ctx); case LZP_TYPE: ctx.putInt("lz", LZP_TYPE); return new LZCodec(ctx); case RANK_TYPE: return new SBRT(SBRT::MODE_RANK, ctx); case SRT_TYPE: return new SRT(ctx); case MTFT_TYPE: return new SBRT(SBRT::MODE_MTF, ctx); case ZRLT_TYPE: return new ZRLT(ctx); case RLT_TYPE: return new RLT(ctx); case EXE_TYPE: return new EXECodec(ctx); case UTF_TYPE: return new UTFCodec(ctx); case PACK_TYPE: return new AliasCodec(ctx); case DNA_TYPE: ctx.putInt("packOnlyDNA", 1); return new AliasCodec(ctx); case MM_TYPE: return new FSDCodec(ctx); case NONE_TYPE: return new NullTransform(ctx); default: std::stringstream ss; ss << "Unknown transform type: '" << functionType << "'"; throw std::invalid_argument(ss.str()); } } template std::string TransformFactory::getName(uint64 functionType) { std::stringstream res; bool first = true; for (int i = 0; i < 8; i++) { const uint64 t = (functionType >> (MAX_SHIFT - ONE_SHIFT * i)) & MASK; if (t == NONE_TYPE) continue; if (first == false) res << '+'; res << getNameToken(t); first = false; } return (first == true) ? getNameToken(NONE_TYPE) : res.str(); } template const char* TransformFactory::getNameToken(uint64 functionType) { switch (functionType) { case DICT_TYPE: return "TEXT"; case BWT_TYPE: return "BWT"; case BWTS_TYPE: return "BWTS"; case ROLZ_TYPE: return "ROLZ"; case ROLZX_TYPE: return "ROLZX"; case LZ_TYPE: return "LZ"; case LZX_TYPE: return "LZX"; case LZP_TYPE: return "LZP"; case ZRLT_TYPE: return "ZRLT"; case RLT_TYPE: return "RLT"; case SRT_TYPE: return "SRT"; case RANK_TYPE: return "RANK"; case MTFT_TYPE: return "MTFT"; case EXE_TYPE: return "EXE"; case PACK_TYPE: return "PACK"; case DNA_TYPE: return "DNA"; case UTF_TYPE: return "UTF"; case MM_TYPE: return "MM"; case NONE_TYPE: return "NONE"; default: std::stringstream ss; ss << "Unknown transform type: '" << functionType << "'"; throw std::invalid_argument(ss.str()); } } } #endif kanzi-cpp-2.6.0/src/transform/TransformSequence.hpp000066400000000000000000000175521525576310500224150ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_TransformSequence #define knz_TransformSequence #include #include #include "../Transform.hpp" #define SKIP_MASK byte(0xFF) namespace kanzi { // Encapsulates a sequence of transforms in a transform template class TransformSequence FINAL : public Transform { public: TransformSequence(Transform* transforms[8], bool deallocate = true); ~TransformSequence(); bool forward(SliceArray& input, SliceArray& output, int length); bool inverse(SliceArray& input, SliceArray& output, int length); // Required encoding output buffer size int getMaxEncodedLength(int srcLen) const; byte getSkipFlags() const { return _skipFlags; } void setSkipFlags(byte flags) { _skipFlags = flags; } int getNbTransforms() const { return _length; } private: Transform* _transforms[8]; // transforms or functions bool _deallocate; // deallocate memory for transforms ? int _length; // number of transforms byte _skipFlags; // skip transforms }; template TransformSequence::TransformSequence(Transform* transforms[8], bool deallocate) { _deallocate = deallocate; _length = 8; _skipFlags = byte(0); for (int i = 7; i >= 0; i--) { _transforms[i] = transforms[i]; if (_transforms[i] == nullptr) _length = i; } if (_length == 0) throw std::invalid_argument("At least one transform required"); } template TransformSequence::~TransformSequence() { if (_deallocate == true) { for (int i = 0; i < 8; i++) { if (_transforms[i] != nullptr) delete _transforms[i]; } } } template bool TransformSequence::forward(SliceArray& input, SliceArray& output, int count) { if (!SliceArray::isValid(input)) throw std::invalid_argument("Invalid input block"); if (!SliceArray::isValid(output)) throw std::invalid_argument("Invalid output block"); if ((count < 0) || (count > input._length - input._index)) return false; _skipFlags = SKIP_MASK; if (count == 0) return true; SliceArray* in = &input; SliceArray* out = &output; SliceArray buffer(nullptr, 0, 0); const int blockSize = count; const int requiredSize = getMaxEncodedLength(blockSize); int swaps = 0; // Process transforms sequentially for (int i = 0; i < _length; i++) { if (_transforms[i] == nullptr) continue; // Check that the output buffer has enough room. If not, allocate a new one. if (out->_length < requiredSize) { if ((out == &input) || (out == &output)) out = &buffer; if (out->_length < requiredSize) { delete[] out->_array; out->_array = new T[requiredSize]; out->_length = requiredSize; } } const int savedIIdx = in->_index; const int savedOIdx = out->_index; // Apply forward transform if (_transforms[i]->forward(*in, *out, count) == false) { // Transform failed. Either it does not apply to this type // of data or a recoverable error occurred => revert in->_index = savedIIdx; out->_index = savedOIdx; continue; } _skipFlags &= ~byte(1 << (7 - i)); count = out->_index - savedOIdx; in->_index = savedIIdx; out->_index = savedOIdx; std::swap(in, out); swaps++; } if ((swaps & 1) == 0) { if ((count > output._length - output._index) || (count > in->_length - in->_index)) { _skipFlags = SKIP_MASK; } else { std::memmove(&output._array[output._index], &in->_array[in->_index], size_t(count)); } } input._index += blockSize; output._index += count; delete[] buffer._array; return _skipFlags != SKIP_MASK; } template bool TransformSequence::inverse(SliceArray& input, SliceArray& output, int count) { if (!SliceArray::isValid(input)) throw std::invalid_argument("Invalid input block"); if (!SliceArray::isValid(output)) throw std::invalid_argument("Invalid output block"); if ((count < 0) || (count > input._length - input._index)) return false; if (count == 0) return true; if (_skipFlags == SKIP_MASK) { if (count > output._length - output._index) return false; std::memmove(&output._array[output._index], &input._array[input._index], size_t(count)); input._index += count; output._index += count; return true; } const int blockSize = count; bool res = true; SliceArray* in = &input; SliceArray* out = &output; SliceArray buffer(nullptr, 0, 0); int swaps = 0; // Process transforms sequentially in reverse order for (int i = _length - 1; i >= 0; i--) { if ((_skipFlags & byte(1 << (7 - i))) != byte(0)) continue; if (_transforms[i] == nullptr) continue; // Check that the output buffer has enough room. If not, allocate a new one. if (out->_length < output._length) { if ((out == &input) || (out == &output)) out = &buffer; if (out->_length < output._length) { delete[] out->_array; out->_array = new T[output._length]; out->_length = output._length; } } const int savedIIdx = in->_index; const int savedOIdx = out->_index; // Apply inverse transform res = _transforms[i]->inverse(*in, *out, count); // All inverse transforms must succeed if (res == false) break; count = out->_index - savedOIdx; in->_index = savedIIdx; out->_index = savedOIdx; std::swap(in, out); swaps++; } if ((res == true) && ((swaps & 1) == 0)) { if ((count > output._length - output._index) || (count > in->_length - in->_index)) res = false; else { std::memmove(&output._array[output._index], &in->_array[in->_index], size_t(count)); } } input._index += blockSize; output._index += count; delete[] buffer._array; return res; } template int TransformSequence::getMaxEncodedLength(int srcLength) const { int requiredSize = srcLength; for (int i = 0; i < _length; i++) { if (_transforms[i] == nullptr) continue; const int nxtSize = _transforms[i]->getMaxEncodedLength(requiredSize); if (nxtSize > requiredSize) requiredSize = nxtSize; } return requiredSize; } } #endif kanzi-cpp-2.6.0/src/transform/UTFCodec.cpp000066400000000000000000000334541525576310500203370ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include #include "UTFCodec.hpp" #include "../Global.hpp" #include "../types.hpp" using namespace kanzi; using namespace std; const int UTFCodec::MIN_BLOCK_SIZE = 1024; const int UTFCodec::LEN_SEQ[256] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, }; bool UTFCodec::forward(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (count < MIN_BLOCK_SIZE) return false; if (!SliceArray::isValid(input)) throw invalid_argument("UTFCodec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("UTFCodec: Invalid output block"); if (output._length - output._index < getMaxEncodedLength(count)) return false; const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; bool mustValidate = true; if (_pCtx != nullptr) { Global::DataType dt = (Global::DataType)_pCtx->getInt("dataType", Global::UNDEFINED); if ((dt != Global::UNDEFINED) && (dt != Global::UTF8)) return false; mustValidate = dt != Global::UTF8; } int start = 0; if ((count >= 3) && (src[0] == kanzi::byte(0xEF)) && (src[1] == kanzi::byte(0xBB)) && (src[2] == kanzi::byte(0xBF))) { // Byte Order Mark (BOM) start = 3; } else { // First (possibly) invalid symbols (due to block truncation). while ((start < 4) && (LEN_SEQ[uint8(src[start])] == 0)) start++; } if ((mustValidate == true) && (validate(&src[start], count - start - 4)) == false) return false; if (_pCtx != nullptr) _pCtx->putInt("dataType", Global::UTF8); // 1-3 bit size + (7 or 11 or 16 or 21) bit payload // 3 MSBs indicate symbol size (limit map size to 22 bits) // 000 -> 7 bits // 001 -> 11 bits // 010 -> 16 bits // 1xx -> 21 bits if (_aliasMap == nullptr) _aliasMap = new uint32[1 << 22]; memset(_aliasMap, 0, size_t(1 << 22) * sizeof(uint32)); vector v; v.reserve(max(count >> 9, 256)); int n = 0; bool res = true; for (int i = start; i < (count - 4); ) { uint32 val; const int s = pack(&src[i], val); res = s != 0; // Validation of longer sequences // Third kanzi::byte in [0x80..0xBF] res &= ((s != 3) || ((src[i + 2] & kanzi::byte(0xC0)) == kanzi::byte(0x80))); // Third and fourth bytes in [0x80..0xBF] res &= ((s != 4) || ((((uint16(src[i + 2]) << 8) | uint16(src[i + 3])) & 0xC0C0) == 0x8080)); // Add to map ? if (_aliasMap[val] == 0) { n++; res &= (n < 32768); #if __cplusplus >= 201103L v.emplace_back(val, 0); #else sdUTF u(val, 0); v.push_back(u); #endif } if (res == false) break; _aliasMap[val]++; i += s; } const int maxTarget = count - (count / 10); if ((res == false) || (n == 0) || ((3 * n + 6) >= maxTarget)) { return false; } for (int i = 0; i < n; i++) v[i].freq = _aliasMap[v[i].val]; // Sort ranks by decreasing frequencies; sort(v.begin(), v.end()); int dstIdx = 2; // Emit map length then map data dst[dstIdx++] = kanzi::byte(n >> 8); dst[dstIdx++] = kanzi::byte(n); int estimate = dstIdx + 6; for (int i = 0; i < n; i++) { estimate += int((i < 128) ? v[i].freq : 2 * v[i].freq); const uint32 s = v[i].val; _aliasMap[s] = (i < 128) ? i : 0x10080 | ((i << 1) & 0xFF00) | (i & 0x7F); dst[dstIdx] = kanzi::byte(s >> 16); dst[dstIdx + 1] = kanzi::byte(s >> 8); dst[dstIdx + 2] = kanzi::byte(s); dstIdx += 3; } if (estimate >= maxTarget) { // Not worth it return false; } // Emit first (possibly invalid) symbols (due to block truncation) for (int i = 0; i < start; i++) dst[dstIdx++] = src[i]; v.clear(); int srcIdx = start; // Emit aliases while (srcIdx < count - 4) { uint32 val; srcIdx += pack(&src[srcIdx], val); const uint32 alias = _aliasMap[val]; dst[dstIdx++] = kanzi::byte(alias); dst[dstIdx] = kanzi::byte(alias >> 8); dstIdx += (alias >> 16); } dst[0] = kanzi::byte(start); dst[1] = kanzi::byte(srcIdx - (count - 4)); // Emit last (possibly invalid) symbols (due to block truncation) while (srcIdx < count) dst[dstIdx++] = src[srcIdx++]; input._index += srcIdx; output._index += dstIdx; return dstIdx < maxTarget; } bool UTFCodec::inverse(SliceArray& input, SliceArray& output, int count) { if (count == 0) return true; if (count < 4) return false; if (!SliceArray::isValid(input)) throw invalid_argument("UTFCodec: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("UTFCodec: Invalid output block"); if (input._index + count > input._length) return false; const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; const int start = int(src[0]) & 0x03; const int adjust = int(src[1]) & 0x03; // adjust end of regular processing const int n = (int(src[2]) << 8) + int(src[3]); // Protect against invalid map size value if ((n == 0) || (n >= 32768) || (3 * n > count - 4)) return false; struct symb { uint32 val; uint8 len; }; symb m[32768]; int srcIdx = 4; // Build inverse mapping for (int i = 0; i < n; i++) { if (srcIdx + 3 > count) return false; int s = (uint32(src[srcIdx]) << 16) | (uint32(src[srcIdx + 1]) << 8) | uint32(src[srcIdx + 2]); const int sl = unpack(s, reinterpret_cast(&m[i].val)); if (sl == 0) return false; m[i].len = uint8(sl); srcIdx += 3; } int dstIdx = 0; const int srcEnd = count - 4 + adjust; const int dstCap = output._length - output._index; const int dstEnd = dstCap - 4; if (dstEnd < 0) return false; if ((srcEnd > count) || (srcIdx + start > srcEnd) || (dstIdx + start > dstCap)) return false; for (int i = 0; i < start; i++) dst[dstIdx++] = src[srcIdx++]; // Emit data if (n <= 128) { // All valid aliases fit in one byte. while (srcIdx < srcEnd) { const uint alias = uint(src[srcIdx++]); if (KANZI_UNLIKELY(alias >= uint(n))) return false; const symb& s = m[alias]; // The symbol length controls the logical output advance, but the // decoder always copies four bytes from the packed symbol value. if (KANZI_UNLIKELY(dstIdx + 4 > dstCap)) return false; memcpy(&dst[dstIdx], &s.val, 4); dstIdx += s.len; } } else { // Decode the next alias and load its value before storing the current // one. This allows independent dictionary lookups to overlap. while (srcIdx < srcEnd) { uint alias = uint(src[srcIdx++]); if (alias >= 128) { if (KANZI_UNLIKELY(srcIdx >= srcEnd)) return false; alias = (uint(src[srcIdx++]) << 7) + (alias & 0x7F); } if (KANZI_UNLIKELY(alias >= uint(n))) return false; const uint32 val0 = m[alias].val; const uint8 len0 = m[alias].len; if (srcIdx >= srcEnd) { if (KANZI_UNLIKELY(dstIdx + 4 > dstCap)) return false; memcpy(&dst[dstIdx], &val0, 4); dstIdx += len0; break; } alias = uint(src[srcIdx++]); if (alias >= 128) { if (KANZI_UNLIKELY(srcIdx >= srcEnd)) return false; alias = (uint(src[srcIdx++]) << 7) + (alias & 0x7F); } if (KANZI_UNLIKELY(alias >= uint(n))) return false; const uint32 val1 = m[alias].val; const uint8 len1 = m[alias].len; const int needed = int(len0) + int(len1) + 4; if (KANZI_UNLIKELY(dstCap - dstIdx < needed)) return false; memcpy(&dst[dstIdx], &val0, 4); dstIdx += len0; memcpy(&dst[dstIdx], &val1, 4); dstIdx += len1; } } if ((srcIdx == srcEnd) && (dstIdx < dstEnd + adjust)) { if ((srcIdx + 4 - adjust > count) || (dstIdx + 4 - adjust > dstCap)) return false; for (int i = 0; i < 4 - adjust; i++) dst[dstIdx++] = src[srcIdx++]; } input._index += srcIdx; output._index += dstIdx; return srcIdx == count; } // A quick partial validation // A more complete validation is done during processing for the remaining cases // (rules for 3 and 4 kanzi::byte sequences) bool UTFCodec::validate(const kanzi::byte block[], int count) { uint freqs0[256] = { 0 }; uint* freqs1 = new uint[65536]; memset(&freqs1[0], 0, 65536 * sizeof(uint)); uint f0[256] = { 0 }; uint f1[256] = { 0 }; uint f3[256] = { 0 }; uint f2[256] = { 0 }; uint8 prv = 0; const uint8* data = reinterpret_cast(&block[0]); const int count4 = count & -4; bool res = true; // Unroll loop for (int i = 0; i < count4; i += 4) { const uint8 cur0 = data[i]; const uint8 cur1 = data[i + 1]; const uint8 cur2 = data[i + 2]; const uint8 cur3 = data[i + 3]; f0[cur0]++; f1[cur1]++; f2[cur2]++; f3[cur3]++; freqs1[(prv * 256) + cur0]++; freqs1[(cur0 * 256) + cur1]++; freqs1[(cur1 * 256) + cur2]++; freqs1[(cur2 * 256) + cur3]++; prv = cur3; } for (int i = count4; i < count; i++) { freqs0[data[i]]++; freqs1[(prv * 256) + data[i]]++; prv = data[i]; } for (int i = 0; i < 256; i++) { freqs0[i] += (f0[i] + f1[i] + f2[i] + f3[i]); } // Valid UTF-8 sequences // See Unicode 16 Standard - UTF-8 Table 3.7 // U+0000..U+007F 00..7F // U+0080..U+07FF C2..DF 80..BF // U+0800..U+0FFF E0 A0..BF 80..BF // U+1000..U+CFFF E1..EC 80..BF 80..BF // U+D000..U+D7FF ED 80..9F 80..BF 80..BF // U+E000..U+FFFF EE..EF 80..BF 80..BF // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF // Check rules for 1 kanzi::byte uint sum = freqs0[0xC0] + freqs0[0xC1]; uint sum2 = 0; for (int i = 0xF5; i <= 0xFF; i++) sum += freqs0[i]; if (sum != 0) { res = false; goto end; } // Check rules for first 2 bytes for (int i = 0; i < 256; i++) { // Exclude < 0xE0A0 || > 0xE0BF if ((i < 0xA0) || (i > 0xBF)) sum += freqs1[0xE0 * 256 + i]; // Exclude < 0xED80 || > 0xEDE9F if ((i < 0x80) || (i > 0x9F)) sum += freqs1[0xED * 256 + i]; // Exclude < 0xF090 || > 0xF0BF if ((i < 0x90) || (i > 0xBF)) sum += freqs1[0xF0 * 256 + i]; // Exclude < 0xF480 || > 0xF48F if ((i < 0x80) || (i > 0x8F)) sum += freqs1[0xF4 * 256 + i]; if ((i < 0x80) || (i > 0xBF)) { // Exclude < 0x??80 || > 0x??BF with ?? in [C2..DF] for (int j = 0xC2; j <= 0xDF; j++) sum += freqs1[j * 256 + i]; // Exclude < 0x??80 || > 0x??BF with ?? in [E1..EC] for (int j = 0xE1; j <= 0xEC; j++) sum += freqs1[j * 256 + i]; // Exclude < 0x??80 || > 0x??BF with ?? in [F1..F3] sum += freqs1[0xF1 * 256 + i]; sum += freqs1[0xF2 * 256 + i]; sum += freqs1[0xF3 * 256 + i]; // Exclude < 0xEE80 || > 0xEEBF sum += freqs1[0xEE * 256 + i]; // Exclude < 0xEF80 || > 0xEFBF sum += freqs1[0xEF * 256 + i]; } else { // Count non-primary bytes sum2 += freqs0[i]; } if (sum != 0) { res = false; break; } } end: delete[] freqs1; // Ad-hoc threshold return (res == true) && (sum2 >= uint(count / 8)); } kanzi-cpp-2.6.0/src/transform/UTFCodec.hpp000066400000000000000000000073151525576310500203410ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_UTFCodec #define knz_UTFCodec #include "../Context.hpp" #include "../Transform.hpp" namespace kanzi { typedef struct ssUTF { uint32 val; uint32 freq; ssUTF(uint32 v, uint32 f) : val(v), freq(f) {} friend bool operator< (ssUTF const& lhs, ssUTF const& rhs) { int r; return ((r = int(lhs.freq - rhs.freq)) != 0) ? r > 0 : lhs.val > rhs.val; } } sdUTF; // UTF8 encoder/decoder class UTFCodec FINAL : public Transform { public: UTFCodec() : _pCtx(nullptr), _aliasMap(nullptr) {} UTFCodec(Context& ctx) : _pCtx(&ctx), _aliasMap(nullptr) {} ~UTFCodec() { delete[] _aliasMap; } bool forward(SliceArray& source, SliceArray& destination, int length); bool inverse(SliceArray& source, SliceArray& destination, int length); int getMaxEncodedLength(int srcLen) const { return srcLen + 8192; } private: static const int MIN_BLOCK_SIZE; static const int LEN_SEQ[256]; Context* _pCtx; uint32* _aliasMap; static bool validate(const byte block[], int count); static int pack(const byte in[], uint32& out); static int unpack(uint32 in, byte out[]); }; inline int UTFCodec::pack(const byte in[], uint32& out) { int s; switch (int(in[0]) >> 4) { case 0: case 1: case 2: case 3: case 4: case 5: case 6: case 7: out = uint32(in[0]); s = 1; break; case 12: case 13: out = (1 << 19) | (uint32(in[0]) << 8) | uint32(in[1]); s = 2; break; case 14: out = (2 << 19) | ((uint32(in[0]) & 0x0F) << 12) | ((uint32(in[1]) & 0x3F) << 6) | (uint32(in[2]) & 0x3F); s = 3; break; case 15: out = (4 << 19) | ((uint32(in[0]) & 0x07) << 18) | ((uint32(in[1]) & 0x3F) << 12) | ((uint32(in[2]) & 0x3F) << 6) | (uint32(in[3]) & 0x3F); s = 4; break; default: out = 0; s = 0; // signal invalid value break; } return s; } inline int UTFCodec::unpack(uint32 in, byte out[]) { int s; switch (in >> 19) { case 0: out[0] = byte(in); s = 1; break; case 1: out[0] = byte(in >> 8); out[1] = byte(in); s = 2; break; case 2: out[0] = byte(((in >> 12) & 0x0F) | 0xE0); out[1] = byte(((in >> 6) & 0x3F) | 0x80); out[2] = byte((in & 0x3F) | 0x80); s = 3; break; case 4: case 5: case 6: case 7: out[0] = byte(((in >> 18) & 0x07) | 0xF0); out[1] = byte(((in >> 12) & 0x3F) | 0x80); out[2] = byte(((in >> 6) & 0x3F) | 0x80); out[3] = byte((in & 0x3F) | 0x80); s = 4; break; default: s = 0; // signal invalid value break; } return s; } } #endif kanzi-cpp-2.6.0/src/transform/ZRLT.cpp000066400000000000000000000153561525576310500175370ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include #include #include #include "../Global.hpp" #include "../Memory.hpp" #include "ZRLT.hpp" using namespace kanzi; using namespace std; bool ZRLT::forward(SliceArray& input, SliceArray& output, int length) { if (length == 0) return true; if (!SliceArray::isValid(input)) throw invalid_argument("ZRLT: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("ZRLT: Invalid output block"); if (output._length - output._index < getMaxEncodedLength(length)) return false; const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; uint srcIdx = 0; uint dstIdx = 0; const uint srcEnd = uint(length); const uint dstEnd = uint(output._length - output._index); const uint srcEnd4 = (length >= 4) ? uint(length - 4) : uint(0); bool res = true; kanzi::byte zeros[4] = { kanzi::byte(0) }; while (srcIdx < srcEnd) { if (src[srcIdx] == kanzi::byte(0)) { uint runLength = 1; while ((srcIdx + runLength < srcEnd4) && (KANZI_MEM_EQ4(&src[srcIdx + runLength], &zeros[0]))) runLength += 4; while ((srcIdx + runLength < srcEnd) && src[srcIdx + runLength] == kanzi::byte(0)) runLength++; srcIdx += runLength; // Encode length runLength++; uint log = uint(Global::_log2(uint32(runLength))); if (KANZI_UNLIKELY(log > dstEnd - dstIdx)) { res = false; break; } // Write every bit as a kanzi::byte except the most significant one while (log >= 4) { const uint32 w = (uint32(((runLength >> (log - 1)) & 1) << 24) | uint32(((runLength >> (log - 2)) & 1) << 16) | uint32(((runLength >> (log - 3)) & 1) << 8) | uint32( (runLength >> (log - 4)) & 1)); BigEndian::writeInt32(&dst[dstIdx], int32(w)); dstIdx += 4; log -= 4; } while (log > 0) { log--; dst[dstIdx++] = kanzi::byte((runLength >> log) & 1); } continue; } const int val = int(src[srcIdx]); const uint needed = (val >= 0xFE) ? uint(2) : uint(1); if (KANZI_UNLIKELY(needed > dstEnd - dstIdx)) { res = false; break; } if (val >= 0xFE) { dst[dstIdx] = kanzi::byte(0xFF); dst[dstIdx + 1] = kanzi::byte(val - 0xFE); dstIdx++; } else { dst[dstIdx] = kanzi::byte(val + 1); } srcIdx++; dstIdx++; } input._index += srcIdx; output._index += dstIdx; return res && (srcIdx == srcEnd); } bool ZRLT::inverse(SliceArray& input, SliceArray& output, int length) { if (length < 0) return false; if (length == 0) return true; if (length > input._length - input._index) return false; if (!SliceArray::isValid(input)) throw invalid_argument("ZRLT: Invalid input block"); if (!SliceArray::isValid(output)) throw invalid_argument("ZRLT: Invalid output block"); const kanzi::byte* src = &input._array[input._index]; kanzi::byte* dst = &output._array[output._index]; uint srcIdx = 0; uint dstIdx = 0; const uint srcEnd = uint(length); const uint dstEnd = uint(output._length - output._index); uint runLength = 0; while (true) { uint val = uint(src[srcIdx]); if (val <= 1) { // Generate the run length bit by bit (but force MSB) runLength = 1; do { runLength += (runLength + val); srcIdx++; if (srcIdx >= srcEnd) goto End; val = uint(src[srcIdx]); } while (val <= 1); runLength--; if (runLength > 0) { if (KANZI_UNLIKELY(runLength >= dstEnd - dstIdx)) goto End; memset(&dst[dstIdx], 0, size_t(runLength)); dstIdx += runLength; runLength = 0; continue; } } // Regular data processing if (KANZI_UNLIKELY(dstIdx >= dstEnd)) return false; if (val != 0xFF) { const uint startIdx = srcIdx; while ((srcIdx + 4 <= srcEnd) && (dstIdx + 4 <= dstEnd)) { const uint32 word = uint32(LittleEndian::readInt32(&src[srcIdx])); // Detect bytes below 2 or equal to 0xFF before lane-wise subtraction. const uint32 invalid = (((word - 0x02020202U) & ~word) | ((~word - 0x01010101U) & word)) & 0x80808080U; if (invalid != 0) break; LittleEndian::writeInt32(&dst[dstIdx], int32(word - 0x01010101U)); srcIdx += 4; dstIdx += 4; } if (srcIdx != startIdx) { if (KANZI_UNLIKELY((srcIdx >= srcEnd) || (dstIdx >= dstEnd))) break; continue; } } if (val == 0xFF) { srcIdx++; if (KANZI_UNLIKELY(srcIdx >= srcEnd)) return false; dst[dstIdx] = kanzi::byte(0xFE + int(src[srcIdx])); } else { dst[dstIdx] = kanzi::byte(val - 1); } srcIdx++; dstIdx++; if (KANZI_UNLIKELY((srcIdx >= srcEnd) || (dstIdx >= dstEnd))) break; } End: if (runLength > 0) { runLength--; // If runLength is not 1, add trailing 0s if (runLength > dstEnd - dstIdx) return false; if (runLength > 0) { memset(&dst[dstIdx], 0, size_t(runLength)); dstIdx += runLength; } } input._index += srcIdx; output._index += dstIdx; return srcIdx == srcEnd; } kanzi-cpp-2.6.0/src/transform/ZRLT.hpp000066400000000000000000000027111525576310500175330ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_ZRLT #define knz_ZRLT #include "../Context.hpp" #include "../Transform.hpp" namespace kanzi { // Zero Run Length Encoding is a simple encoding algorithm by Wheeler // closely related to Run Length Encoding. The main difference is // that only runs of 0 values are processed. Also, the length is // encoded in a different way (each digit in a different byte) // This algorithm is well adapted to process post BWT/MTFT data. class ZRLT FINAL : public Transform { public: ZRLT() {} ZRLT(Context&) {} ~ZRLT() {} bool forward(SliceArray& pSrc, SliceArray& pDst, int length); bool inverse(SliceArray& pSrc, SliceArray& pDst, int length); // Required encoding output buffer size unknown => guess int getMaxEncodedLength(int srcLen) const { return srcLen; } }; } #endif kanzi-cpp-2.6.0/src/types.hpp000066400000000000000000000221101525576310500160640ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_types #define knz_types #if defined(_MSC_VER) && _MSC_VER < 1600 // Visual Studio < 2010: no stdint.h typedef unsigned char uint8_t; typedef signed char int8_t; typedef unsigned short uint16_t; typedef short int16_t; typedef unsigned int uint32_t; typedef int int32_t; typedef unsigned __int64 uint64_t; typedef __int64 int64_t; #else #if (__cplusplus >= 201103L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201103L) // C++11 or later #include #include #else // C++98 / C++03 #include #include #endif #endif #if defined(_MSC_VER) #if _MSC_VER < 1900 // snprintf macro for MSVC < 2015 #define snprintf _snprintf #endif #if defined(_M_X64) && !defined(__x86_64__) #define __x86_64__ _M_X64 #endif #if defined(_M_IX86) && !defined(__i386__) #define __i386__ _M_IX86 #endif #endif #if defined(_MSC_VER) #include #define popcount __popcnt #else #if defined(__INTEL_COMPILER) #include #define popcount _popcnt32 #else #define popcount __builtin_popcount #endif #endif #ifdef __SSE__ #include #endif #ifdef __SSE2__ #include #endif #ifdef __SSE3__ #include #endif #ifdef __SSE4_1__ #include #endif #ifdef __AVX__ #include #endif #ifdef __AVX2__ #include #endif /* Visual Studio 2022 17.2 MSVC++ 14.28 _MSC_VER == 1932 Visual Studio 2022 17.1 MSVC++ 14.28 _MSC_VER == 1931 Visual Studio 2022 17.0 MSVC++ 14.28 _MSC_VER == 1930 Visual Studio 2019 version 16.10, 16.11 MSVC++ 14.28 _MSC_VER == 1929 Visual Studio 2019 version 16.8, 16.9 MSVC++ 14.28 _MSC_VER == 1928 Visual Studio 2019 version 16.7 MSVC++ 14.27 _MSC_VER == 1927 Visual Studio 2019 version 16.6 MSVC++ 14.26 _MSC_VER == 1926 Visual Studio 2019 version 16.5 MSVC++ 14.25 _MSC_VER == 1925 Visual Studio 2019 Update 4 MSVC++ 14.24 _MSC_VER == 1924 Visual Studio 2019 Update 3 MSVC++ 14.21 _MSC_VER == 1923 Visual Studio 2019 Update 2 MSVC++ 14.21 _MSC_VER == 1922 Visual Studio 2019 Update 1 MSVC++ 14.21 _MSC_VER == 1921 Visual Studio 2019 MSVC++ 14.20 _MSC_VER == 1920 Visual Studio 2017 Update 9 MSVC++ 14.16 _MSC_VER == 1916 Visual Studio 2017 Update 8 MSVC++ 14.15 _MSC_VER == 1915 Visual Studio 2017 Update 7 MSVC++ 14.14 _MSC_VER == 1914 Visual Studio 2017 Update 6 MSVC++ 14.13 _MSC_VER == 1913 Visual Studio 2017 Update 5 MSVC++ 14.12 _MSC_VER == 1912 Visual Studio 2017 Update 3&4 MSVC++ 14.11 _MSC_VER == 1911 Visual Studio 2017 MSVC++ 14.10 _MSC_VER == 1910 Visual Studio 2015 MSVC++ 14 _MSC_VER == 1900 Visual Studio 2013 MSVC++ 12 _MSC_VER == 1800 Visual Studio 2012 MSVC++ 11 _MSC_VER == 1700 Visual Studio 2010 MSVC++ 10 _MSC_VER == 1600 Visual Studio 2008 MSVC++ 9 _MSC_VER == 1500 Visual Studio 2005 MSVC++ 8 _MSC_VER == 1400 Visual Studio 2003 Beta MSVC++ 7.1 _MSC_VER == 1310 Visual Studio 2002 MSVC++ 7 _MSC_VER == 1300 Visual Studio MSVC++ 6.0 _MSC_VER == 1200 Visual Studio MSVC++ 5 _MSC_VER == 1100 Visual Studio MSVC++ 4.2 _MSC_VER == 1020 Visual Studio MSVC++ 4.1 _MSC_VER == 1010 Visual Studio MSVC++ 4 _MSC_VER == 1000 Visual Studio MSVC++ 2 _MSC_VER == 900 Visual Studio MSVC++ 1 _MSC_VER == 800 */ #ifdef _MSC_VER #if _MSC_VER >= 1930 #define _MSC_VER_STR 2022 #elif _MSC_VER >= 1920 #define _MSC_VER_STR 2019 #elif _MSC_VER >= 1910 #define _MSC_VER_STR 2017 #elif _MSC_VER == 1900 #define _MSC_VER_STR 2015 #elif _MSC_VER == 1800 #define _MSC_VER_STR 2013 #elif _MSC_VER == 1700 #define _MSC_VER_STR 2012 #elif _MSC_VER == 1600 #define _MSC_VER_STR 2010 #elif _MSC_VER == 1500 #define _MSC_VER_STR 2008 #elif _MSC_VER == 1400 #define _MSC_VER_STR 2005 #elif _MSC_VER == 1300 #define _MSC_VER_STR 2003 #elif _MSC_VER == 1200 #define _MSC_VER_STR 2002 #endif #endif // Notice: in Visual Studio (prior to VS2017 version 15.7) // __cplusplus always defaults to 199711L (aka C++98) !!! (unless // the extra option /Zc:__cplusplus is added to the command line). // Otherwise, using the _MSVC_LANG macro returns the proper C++ version. #if (__cplusplus >= 201103L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201103L) // C++ 11 or higher #define FINAL final #define NOEXCEPT noexcept #else #define FINAL #define NOEXCEPT throw() #if defined(_MSC_VER) #if _MSC_VER < 1300 typedef signed char int8_t; typedef signed short int16_t; typedef signed int int32_t; typedef unsigned char uint8_t; typedef unsigned short uint16_t; typedef unsigned int uint32_t; #else typedef signed __int8 int8_t; typedef signed __int16 int16_t; typedef signed __int32 int32_t; typedef unsigned __int8 uint8_t; typedef unsigned __int16 uint16_t; typedef unsigned __int32 uint32_t; typedef signed __int64 int64_t; typedef unsigned __int64 uint64_t; #endif #else // If stdint.h did not provide fixed-width types, define them here. #ifndef INT64_MAX typedef signed char int8_t; typedef signed short int16_t; typedef signed int int32_t; typedef unsigned char uint8_t; typedef unsigned short uint16_t; typedef unsigned int uint32_t; typedef signed long long int64_t; typedef unsigned long long uint64_t; #endif #endif #if !defined(nullptr) #define nullptr NULL #endif #endif namespace kanzi { #if (__cplusplus >= 201703L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) using byte = std::byte; #else typedef uint8_t byte; #endif typedef int8_t int8; typedef uint8_t uint8; typedef int16_t int16; typedef uint16_t uint16; typedef int32_t int32; typedef uint32_t uint32; typedef uint32_t uint; typedef int64_t int64; typedef uint64_t uint64; } #if defined(__MINGW32__) #define PATH_SEPARATOR '/' #elif defined(WIN32) || defined(_WIN32) || defined(_WIN64) #define PATH_SEPARATOR '\\' #else #define PATH_SEPARATOR '/' #endif // Likely / unlikely macros #if defined(__GNUC__) || defined(__clang__) #ifndef KANZI_LIKELY #define KANZI_LIKELY(x) __builtin_expect(!!(x), 1) #endif #ifndef KANZI_UNLIKELY #define KANZI_UNLIKELY(x) __builtin_expect(!!(x), 0) #endif #else #ifndef KANZI_LIKELY #define KANZI_LIKELY(x) (x) #endif #ifndef KANZI_UNLIKELY #define KANZI_UNLIKELY(x) (x) #endif #endif // Force inline macro #if defined(__GNUC__) || defined(__clang__) #define KANZI_ALWAYS_INLINE inline __attribute__((always_inline)) #elif defined(_MSC_VER) #define KANZI_ALWAYS_INLINE __forceinline #else #define KANZI_ALWAYS_INLINE inline #endif #if defined(_MSC_VER) #define KANZI_ALIGNED_(x) __declspec(align(x)) #elif defined(__GNUC__) #define KANZI_ALIGNED_(x) __attribute__ ((aligned(x))) #endif #endif kanzi-cpp-2.6.0/src/util/000077500000000000000000000000001525576310500151705ustar00rootroot00000000000000kanzi-cpp-2.6.0/src/util/Clock.hpp000066400000000000000000000024471525576310500167430ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Clock #define knz_Clock #include "WallTimer.hpp" namespace kanzi { class Clock { private: WallTimer _timer; WallTimer::TimeData _start; WallTimer::TimeData _stop; public: Clock() { start(); _stop = _start; } void start() { _start = _timer.getCurrentTime(); } void stop() { _stop = _timer.getCurrentTime(); } double elapsed() const { // In millisec return WallTimer::calculateDifference(_start, _stop); } }; } #endif kanzi-cpp-2.6.0/src/util/Printer.hpp000066400000000000000000000043411525576310500173260ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_Printer #define knz_Printer #ifdef CONCURRENCY_ENABLED #include #endif namespace kanzi { // Thread safe printer class Printer { public: Printer(std::ostream& os) { _os = &os; } ~Printer() { try { _os->flush(); } catch (const std::exception&) { // Ignore: best effort } } void print(const char* msg, bool print) { if ((print == true) && (msg != nullptr)) { #ifdef CONCURRENCY_ENABLED std::lock_guard lock(getMutex()); #endif (*_os) << msg; } } void println(const char* msg, bool print) { if ((print == true) && (msg != nullptr)) { #ifdef CONCURRENCY_ENABLED std::lock_guard lock(getMutex()); #endif (*_os) << msg << std::endl; } } void print(const std::string& msg, bool print) { if (print == true) { #ifdef CONCURRENCY_ENABLED std::lock_guard lock(getMutex()); #endif (*_os) << msg; } } void println(const std::string& msg, bool print) { if (print == true) { #ifdef CONCURRENCY_ENABLED std::lock_guard lock(getMutex()); #endif (*_os) << msg << std::endl; } } private: #ifdef CONCURRENCY_ENABLED static std::mutex& getMutex() { static std::mutex mtx; return mtx; } #endif std::ostream* _os; }; } #endif kanzi-cpp-2.6.0/src/util/WallTimer.cpp000066400000000000000000000013361525576310500175770ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #include "WallTimer.hpp" #if defined(KNZ_USE_WINDOWS_QPC) LARGE_INTEGER WallTimer::_frequency = {}; bool WallTimer::_initialized = false; #endif kanzi-cpp-2.6.0/src/util/WallTimer.hpp000066400000000000000000000067351525576310500176140ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_WallTimer #define knz_WallTimer #include "../types.hpp" // Portable wall timer // 1. Detect Standard and Platform #if __cplusplus >= 201103L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201103L) #include #define KNZ_USE_CHRONO #elif defined(_WIN32) || defined(_WIN64) #include #define KNZ_USE_WINDOWS_QPC #else #include #define KNZ_USE_POSIX_GETTIMEOFDAY #endif class WallTimer { public: struct TimeData { #if defined(KNZ_USE_CHRONO) std::chrono::steady_clock::time_point value; #elif defined(KNZ_USE_WINDOWS_QPC) LARGE_INTEGER value; #elif defined(KNZ_USE_POSIX_GETTIMEOFDAY) struct timeval value; #endif // Converts the internal timestamp into total milliseconds. #if defined(KNZ_USE_CHRONO) kanzi::uint64 to_ms() const { return static_cast(std::chrono::duration(value.time_since_epoch()).count()); } #elif defined(KNZ_USE_WINDOWS_QPC) double to_ms(long long win_freq = 1) const { return static_cast(value.QuadPart) * 1000.0 / win_freq; } #elif defined(KNZ_USE_POSIX_GETTIMEOFDAY) double to_ms() const { return (static_cast(value.tv_sec) * 1000.0) + (value.tv_usec / 1000.0); } #endif }; WallTimer() { #if defined(KNZ_USE_WINDOWS_QPC) if (_initialized == false) { QueryPerformanceFrequency(&_frequency); _initialized = true; } #endif _start = getCurrentTime(); } TimeData getCurrentTime() const { TimeData now; #if defined(KNZ_USE_CHRONO) now.value = std::chrono::steady_clock::now(); #elif defined(KNZ_USE_WINDOWS_QPC) QueryPerformanceCounter(&now.value); #elif defined(KNZ_USE_POSIX_GETTIMEOFDAY) gettimeofday(&now.value, 0); #endif return now; } static double calculateDifference(const TimeData& start, const TimeData& end) { #if defined(KNZ_USE_CHRONO) return std::chrono::duration(end.value - start.value).count(); #elif defined(KNZ_USE_WINDOWS_QPC) if (_initialized == false) { QueryPerformanceFrequency(&_frequency); _initialized = true; } return static_cast(end.value.QuadPart - start.value.QuadPart) * 1000.0 / static_cast(_frequency.QuadPart); #elif defined(KNZ_USE_POSIX_GETTIMEOFDAY) const double sec = double(end.value.tv_sec - start.value.tv_sec); const double usec = double(end.value.tv_usec - start.value.tv_usec); return (sec * 1000.0) + (usec / 1000.0); #endif } double elapsed_ms() const { return calculateDifference(_start, getCurrentTime()); } private: TimeData _start; #if defined(KNZ_USE_WINDOWS_QPC) static LARGE_INTEGER _frequency; static bool _initialized; #endif }; #endif kanzi-cpp-2.6.0/src/util/XXHash.hpp000066400000000000000000000156101525576310500170470ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_XXHash32 #define knz_XXHash32 #include #include "../Memory.hpp" namespace kanzi { // XXHash is an extremely fast hash algorithm. It was written by Yann Collet. // Original source code: https://github.com/Cyan4973/xxHash class XXHash32 { private: #if __cplusplus >= 201103L static constexpr uint32 PRIME32_1 = uint32(-1640531535); static constexpr uint32 PRIME32_2 = uint32(-2048144777); static constexpr uint32 PRIME32_3 = uint32(-1028477379); static constexpr uint32 PRIME32_4 = uint32(668265263); static constexpr uint32 PRIME32_5 = uint32(374761393); #else static const uint32 PRIME32_1 = uint32(-1640531535); static const uint32 PRIME32_2 = uint32(-2048144777); static const uint32 PRIME32_3 = uint32(-1028477379); static const uint32 PRIME32_4 = uint32(668265263); static const uint32 PRIME32_5 = uint32(374761393); #endif uint32 _seed; uint32 round(uint32 acc, int32 val) const; public: XXHash32() { _seed = uint32(time(nullptr)); } XXHash32(uint32 seed) : _seed(seed) {} ~XXHash32(){} void setSeed(uint32 seed) { _seed = seed; } uint32 hash(const byte data[], int length) const; }; inline uint32 XXHash32::hash(const byte data[], int length) const { uint32 h32; int idx = 0; if (length >= 16) { const int end16 = length - 16; uint32 v1 = _seed + PRIME32_1 + PRIME32_2; uint32 v2 = _seed + PRIME32_2; uint32 v3 = _seed; uint32 v4 = _seed - PRIME32_1; do { v1 = round(v1, LittleEndian::readInt32(&data[idx])); v2 = round(v2, LittleEndian::readInt32(&data[idx + 4])); v3 = round(v3, LittleEndian::readInt32(&data[idx + 8])); v4 = round(v4, LittleEndian::readInt32(&data[idx + 12])); idx += 16; } while (idx <= end16); h32 = ((v1 << 1) | (v1 >> 31)); h32 += ((v2 << 7) | (v2 >> 25)); h32 += ((v3 << 12) | (v3 >> 20)); h32 += ((v4 << 18) | (v4 >> 14)); } else { h32 = _seed + PRIME32_5; } h32 += uint32(length); while (idx <= length - 4) { h32 += (uint32(LittleEndian::readInt32(&data[idx])) * PRIME32_3); h32 = ((h32 << 17) | (h32 >> 15)) * PRIME32_4; idx += 4; } while (idx < length) { h32 += ((uint32(data[idx]) & 0xFF) * PRIME32_5); h32 = ((h32 << 11) | (h32 >> 21)) * PRIME32_1; idx++; } h32 ^= (h32 >> 15); h32 *= PRIME32_2; h32 ^= (h32 >> 13); h32 *= PRIME32_3; return h32 ^ (h32 >> 16); } inline uint32 XXHash32::round(uint32 acc, int32 val) const { acc += (uint32(val) * PRIME32_2); return ((acc << 13) | (acc >> 19)) * PRIME32_1; } class XXHash64 { private: #if __cplusplus >= 201103L static constexpr uint64 PRIME64_1 = uint64(0x9E3779B185EBCA87); static constexpr uint64 PRIME64_2 = uint64(0xC2B2AE3D27D4EB4F); static constexpr uint64 PRIME64_3 = uint64(0x165667B19E3779F9); static constexpr uint64 PRIME64_4 = uint64(0x85EBCA77C2b2AE63); static constexpr uint64 PRIME64_5 = uint64(0x27D4EB2F165667C5); #else static const uint64 PRIME64_1 = uint64(0x9E3779B185EBCA87); static const uint64 PRIME64_2 = uint64(0xC2B2AE3D27D4EB4F); static const uint64 PRIME64_3 = uint64(0x165667B19E3779F9); static const uint64 PRIME64_4 = uint64(0x85EBCA77C2b2AE63); static const uint64 PRIME64_5 = uint64(0x27D4EB2F165667C5); #endif int64 _seed; uint64 round(uint64 acc, uint64 val) const; uint64 mergeRound(uint64 acc, uint64 val) const; public: XXHash64() { _seed = int64(time(nullptr)); } XXHash64(int64 seed) : _seed(seed) {} ~XXHash64(){} void setSeed(int64 seed) { _seed = seed; } uint64 hash(const byte data[], int length) const; }; inline uint64 XXHash64::hash(const byte data[], int length) const { uint64 h64; int idx = 0; if (length >= 32) { const int length32 = length - 32; uint64 v1 = _seed + PRIME64_1 + PRIME64_2; uint64 v2 = _seed + PRIME64_2; uint64 v3 = _seed; uint64 v4 = _seed - PRIME64_1; do { v1 = round(v1, uint64(LittleEndian::readLong64(&data[idx]))); v2 = round(v2, uint64(LittleEndian::readLong64(&data[idx + 8]))); v3 = round(v3, uint64(LittleEndian::readLong64(&data[idx + 16]))); v4 = round(v4, uint64(LittleEndian::readLong64(&data[idx + 24]))); idx += 32; } while (idx <= length32); h64 = ((v1 << 1) | (v1 >> 31)) + ((v2 << 7) | (v2 >> 25)) + ((v3 << 12) | (v3 >> 20)) + ((v4 << 18) | (v4 >> 14)); h64 = mergeRound(h64, v1); h64 = mergeRound(h64, v2); h64 = mergeRound(h64, v3); h64 = mergeRound(h64, v4); } else { h64 = _seed + PRIME64_5; } h64 += length; while (idx+8 <= length) { h64 ^= round(0, uint64(LittleEndian::readLong64(&data[idx]))); h64 = ((h64 << 27) | (h64 >> 37)) * PRIME64_1 + PRIME64_4; idx += 8; } while (idx+4 <= length) { h64 ^= (uint32(LittleEndian::readInt32(&data[idx])) * PRIME64_1); h64 = ((h64 << 23) | (h64 >> 41)) * PRIME64_2 + PRIME64_3; idx += 4; } while (idx < length) { h64 ^= (uint64(data[idx] & byte(0xFF)) * PRIME64_5); h64 = ((h64 << 11) | (h64 >> 53)) * PRIME64_1; idx++; } // Finalize h64 ^= (h64 >> 33); h64 *= PRIME64_2; h64 ^= (h64 >> 29); h64 *= PRIME64_3; return h64 ^ (h64 >> 32); } inline uint64 XXHash64::round(uint64 acc, uint64 val) const { acc += (val*PRIME64_2); return ((acc << 31) | (acc >> 33)) * PRIME64_1; } inline uint64 XXHash64::mergeRound(uint64 acc, uint64 val) const { acc ^= round(0, val); return acc*PRIME64_1 + PRIME64_4; } } #endif kanzi-cpp-2.6.0/src/util/fixedbuf.hpp000066400000000000000000000072061525576310500175020ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_fixedbuf #define knz_fixedbuf #include #include #include #include #include "../SliceArray.hpp" #include "../types.hpp" // Ahem ... Visual Studio // This code is required because Microsoft cannot bother to implement streambuf::pubsetbuf(). // Also On libstdc++, pubsetbuf() silently ignores the supplied buffer and leaves internal pointers null. class ifixedbuf : public std::streambuf { public: ifixedbuf(char* data, std::size_t size) { // Always manually set the read pointers. // pubsetbuf() is unreliable on libstdc++, and MSVC doesn't implement it. this->setg(data, data, data + size); } }; class ofixedbuf : public std::streambuf { public: ofixedbuf(char* data, std::size_t size) { // Always set buffer manually - pubsetbuf is useless on libstdc++ this->setp(data, data + size); } std::size_t written() const { return this->pptr() - this->pbase(); } }; class growable_ofixedbuf FINAL : public std::streambuf { public: explicit growable_ofixedbuf(kanzi::SliceArray* data) : _data(data) { this->setp(reinterpret_cast(_data->_array), reinterpret_cast(_data->_array) + _data->_length); } protected: virtual int_type overflow(int_type c) { if (traits_type::eq_int_type(c, traits_type::eof())) return traits_type::not_eof(c); _grow(std::size_t(this->pptr() - this->pbase()) + 1); *this->pptr() = traits_type::to_char_type(c); this->pbump(1); return c; } virtual std::streamsize xsputn(const char* s, std::streamsize n) { if (n <= 0) return 0; const std::size_t written = std::size_t(this->pptr() - this->pbase()); const std::size_t required = written + std::size_t(n); if (required > std::size_t(this->epptr() - this->pbase())) _grow(required); std::memcpy(this->pptr(), s, std::size_t(n)); this->pbump(int(n)); return n; } private: kanzi::SliceArray* _data; void _grow(std::size_t required) { const std::size_t current = std::size_t(_data->_length); const std::size_t written = std::size_t(this->pptr() - this->pbase()); const std::size_t grown = current + (std::max)(current >> 2, std::size_t(1) << 20); const std::size_t newSize = (std::max)(required, (std::max)(grown, std::size_t(1024))); kanzi::byte* buf = new kanzi::byte[newSize]; if (written > 0) std::memcpy(buf, _data->_array, written); delete[] _data->_array; _data->_array = buf; _data->_length = int(newSize); this->setp(reinterpret_cast(_data->_array), reinterpret_cast(_data->_array) + _data->_length); this->pbump(int(written)); } }; class iofixedbuf : public std::streambuf { public: iofixedbuf(char* data, std::size_t size) { this->setg(data, data, data + size); this->setp(data, data + size); } }; #endif kanzi-cpp-2.6.0/src/util/strings.hpp000066400000000000000000000076341525576310500174040ustar00rootroot00000000000000/* Copyright 2011-2026 Frederic Langlet Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. you may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ #pragma once #ifndef knz_strings #define knz_strings #include #include #include #include #include #if __cplusplus < 201103L // to_string() not available before C++ 11 template std::string to_string(T value) { std::ostringstream os; os << value; return os.str(); } #define TOSTR(v) to_string(v) #else #define TOSTR(v) std::to_string(v) #endif inline void to_binary(int num, char* buffer, int length) { for (int i = length - 2; i >= 0; i--) { buffer[i] = (num & 1) ? '1' : '0'; num >>= 1; } buffer[length - 1] = '\0'; } // trim from end of string (right) inline std::string& rtrim(std::string& s) { static const char* whitespaces = " \t\f\v\n\r"; std::size_t pos = s.find_last_not_of(whitespaces); if (pos == std::string::npos) s.clear(); else s.erase(pos + 1); return s; } // trim from beginning of string (left) inline std::string& ltrim(std::string& s) { static const char* whitespaces = " \t\f\v\n\r"; std::size_t pos = s.find_first_not_of(whitespaces); if (pos == std::string::npos) s.clear(); else s.erase(0, pos); return s; } // Ensures that the function works on platforms where char is signed inline char safeToUpper(char c) { return static_cast(::toupper(static_cast(c))); } // trim from both ends of string (right then left) inline std::string& trim(std::string& s) { return ltrim(rtrim(s)); } inline void tokenize(const std::string& str, std::vector& v, char token) { std::istringstream ss(str); std::string s; while (getline(ss, s, token)) v.push_back(s); } inline std::string escapeJSONString(const std::string& src) { std::stringstream ss; for (size_t i = 0; i < src.size(); i++) { const unsigned char c = static_cast(src[i]); switch (c) { case '"': ss << "\\\""; break; case '\\': ss << "\\\\"; break; case '\b': ss << "\\b"; break; case '\f': ss << "\\f"; break; case '\n': ss << "\\n"; break; case '\r': ss << "\\r"; break; case '\t': ss << "\\t"; break; default: if (c < 0x20) { ss << "\\u00"; ss << std::uppercase << std::hex << std::setw(2) << std::setfill('0') << int(c); ss << std::dec; } else { ss << char(c); } } } return ss.str(); } inline std::string formatSize(double size) { std::stringstream ss; std::string s; if (size >= double(1 << 30)) { size /= double(1024 * 1024 * 1024); ss << std::fixed << std::setprecision(2) << size << " GiB"; s = ss.str(); } else if (size >= double(1 << 20)) { size /= double(1024 * 1024); ss << std::fixed << std::setprecision(2) << size << " MiB"; s = ss.str(); } else if (size >= double(1 << 10)) { size /= double(1024); ss << std::fixed << std::setprecision(2) << size << " KiB"; s = ss.str(); } else { ss << size; s = ss.str(); } return s; } inline std::string formatSize(const std::string& input) { return formatSize(atof(input.c_str())); } #endif